diff --git a/.github/ci-coverage-allowlist.yml b/.github/ci-coverage-allowlist.yml index f7a785b3b80..32232de381c 100644 --- a/.github/ci-coverage-allowlist.yml +++ b/.github/ci-coverage-allowlist.yml @@ -4,6 +4,11 @@ description: >- by a job nor listed here, so every entry below is a decision on the record. test_paths: + - reason: >- + The Rust/Python parity harness is run manually through its local CLI. Recorded replay, + fixture generation, and harness checks are intentionally outside pull request CI + paths: + - tests/rust-python-harness - reason: >- What is left of the caching suite in tests/local_testing that runs nowhere. Every job that globs that directory either deselects it (local_testing_part1 and part2 carry `-k "... and diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index a13dd4c04b0..643ad985251 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -27,7 +27,7 @@ rand = "0.8" reqwest = { version = "0.12", default-features = false, features = ["blocking", "json", "rustls-tls", "http2", "stream"] } rstest = "0.26.1" serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["float_roundtrip"] } sha2 = "0.10" subtle = "2" thiserror = "2.0" diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index b26a7925e8a..641a019476e 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -14,7 +14,7 @@ const AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGE const AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: &str = "2024-11-30"; const AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: i64 = 96; -const AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS: &[&str] = &["pages"]; +const AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS: &[&str] = &["pages", "features"]; pub struct AzureAiOcrConfig; pub struct AzureDocumentIntelligenceOcrConfig; @@ -192,6 +192,46 @@ fn normalize_pages_param(pages: &Value) -> Result, Error> { } } +fn feature_token_is_valid(token: &str) -> bool { + let Some((first, rest)) = token.as_bytes().split_first() else { + return false; + }; + first.is_ascii_alphabetic() && rest.iter().all(u8::is_ascii_alphanumeric) +} + +fn invalid_features_error(features: &Value) -> Error { + Error::InvalidRequest(format!( + "Invalid `features` for Azure Document Intelligence: {features:?}. Expected a list of feature names or a comma-separated string like 'keyValuePairs' or 'keyValuePairs,languages'." + )) +} + +fn normalize_features_param(features: &Value) -> Result, Error> { + let normalized = match features { + Value::String(value) => value + .split(',') + .map(str::trim) + .collect::>() + .join(","), + Value::Array(values) if values.is_empty() => return Ok(None), + Value::Array(values) => values + .iter() + .map(Value::as_str) + .collect::>>() + .ok_or_else(|| invalid_features_error(features))? + .into_iter() + .map(str::trim) + .collect::>() + .join(","), + _ => return Err(invalid_features_error(features)), + }; + + if normalized.split(',').all(feature_token_is_valid) { + Ok(Some(normalized)) + } else { + Err(invalid_features_error(features)) + } +} + pub fn complete_document_intelligence_url( api_base: Option<&str>, model: &str, @@ -213,6 +253,13 @@ pub fn complete_document_intelligence_url( url.push_str(&normalized); } + if let Some(features) = optional_params.get("features") + && let Some(normalized) = normalize_features_param(features)? + { + url.push_str("&features="); + url.push_str(&normalized); + } + Ok(url) } @@ -475,6 +522,103 @@ mod tests { ); } + #[test] + fn document_intelligence_url_normalizes_features() { + let params = serde_json::Map::from_iter([( + "features".to_string(), + json!("keyValuePairs, languages"), + )]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&features=keyValuePairs,languages" + ); + } + + #[test] + fn document_intelligence_url_combines_pages_and_feature_list() { + let params = serde_json::Map::from_iter([ + ("pages".to_string(), json!([0, 1, 2])), + ( + "features".to_string(), + json!([" keyValuePairs ", "languages"]), + ), + ]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&pages=1,2,3&features=keyValuePairs,languages" + ); + } + + #[test] + fn document_intelligence_url_omits_empty_feature_list() { + let params = serde_json::Map::from_iter([("features".to_string(), json!([]))]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30" + ); + } + + #[test] + fn document_intelligence_url_rejects_invalid_features() { + for features in [ + json!("keyValuePairs&pages=9"), + json!(""), + json!(["keyValuePairs", 1]), + json!({"feature": "keyValuePairs"}), + ] { + let params = serde_json::Map::from_iter([("features".to_string(), features.clone())]); + let error = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect_err("invalid features must fail"); + + assert!( + matches!(error, Error::InvalidRequest(message) if message.contains("Invalid `features`")), + "features={features:?}" + ); + } + } + + #[test] + fn document_intelligence_maps_features() { + let params = Map::from_iter([ + ("features".to_string(), json!(["keyValuePairs"])), + ("unsupported".to_string(), json!(true)), + ]); + + assert_eq!( + AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.map_ocr_params(¶ms), + Map::from_iter([("features".to_string(), json!(["keyValuePairs"]))]) + ); + } + #[test] fn document_intelligence_request_uses_base64_source_for_data_uri() { let body = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 914e2e1e033..76c298abf89 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -59,3 +59,41 @@ pub(crate) fn register(module: &Bound<'_, PyModule>) -> PyResult<()> { module.add("RustBridgeDeclined", py.get_type::())?; module.add("RustUpstreamError", py.get_type::()) } + +pub(crate) fn ocr_error_to_pyerr(err: Error) -> PyErr { + match err { + Error::MissingField("document_url" | "image_url") => { + PyValueError::new_err("Document URL is required") + } + Error::Http { status, body } => RustUpstreamError::new_err((status, body)), + other => core_error_to_pyerr(other), + } +} + +#[cfg(test)] +mod ocr_error_tests { + use super::*; + + #[test] + fn ocr_errors_preserve_python_validation_and_provider_details() { + Python::initialize(); + Python::attach(|py| { + for field in ["document_url", "image_url"] { + let mapped = ocr_error_to_pyerr(Error::MissingField(field)); + assert!(mapped.is_instance_of::(py)); + assert_eq!(mapped.value(py).to_string(), "Document URL is required"); + } + let mapped = ocr_error_to_pyerr(Error::Http { + status: 429, + body: r#"{"message":"rate limited"}"#.to_string(), + }); + assert!(mapped.is_instance_of::(py)); + let args: (u16, String) = mapped + .value(py) + .getattr("args") + .and_then(|args| args.extract()) + .expect("OCR failures retain status and unprefixed provider message"); + assert_eq!(args, (429, r#"{"message":"rate limited"}"#.to_string())); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs index 5588c400972..5cc8804238b 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -5,7 +5,7 @@ use litellm_ai_gateway::io::ocr::{OcrRequest, ocr as run_ocr}; use pyo3::prelude::*; use serde_json::Value; -use crate::errors::core_error_to_pyerr; +use crate::errors::ocr_error_to_pyerr; use crate::marshal::{RouteOptions, RouteOptionsInputs, object_or_empty}; fn prepare_ocr( @@ -69,5 +69,5 @@ bridge_route! { timeout_seconds: Option, }, prepare = prepare_ocr, - errors = core_error_to_pyerr, + errors = ocr_error_to_pyerr, } diff --git a/pyproject.toml b/pyproject.toml index d0e5723d1cb..161994635b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -174,6 +174,8 @@ litellm-proxy = "litellm.proxy.client.cli:cli" [dependency-groups] dev = [ "diff-cover==9.7.2", + "hypothesis==6.165.10", + "reportlab==5.0.1", "basedpyright==1.39.7", "keyring==25.7.0", "pytest==9.0.3", diff --git a/tests/code_coverage_tests/liccheck.ini b/tests/code_coverage_tests/liccheck.ini index f5a2fb4b14c..9103d913c36 100644 --- a/tests/code_coverage_tests/liccheck.ini +++ b/tests/code_coverage_tests/liccheck.ini @@ -172,6 +172,7 @@ pylint: >=3.3.9 # GPLv2 license langchain-mcp-adapters: >=0.2.1 # MIT License langgraph: >=1.0.10 # MIT License langgraph-prebuilt: >=1.0.8 # MIT License - https://github.com/langchain-ai/langgraph/blob/main/LICENSE +hypothesis: >=6.165.10 # MPL 2.0 license pytest-rerunfailures: >=15.1 # MPL 2.0 license pytest-recording: >=0.13.4 # MIT license expression: >=5.6.0 # MIT License - https://github.com/cognitedata/Expression/blob/main/LICENSE diff --git a/tests/rust-python-harness/README.md b/tests/rust-python-harness/README.md index a34e1a1ab73..77df4a24dd3 100644 --- a/tests/rust-python-harness/README.md +++ b/tests/rust-python-harness/README.md @@ -1,148 +1,105 @@ -# Rust ↔ Python SDK parity harness +# Rust/Python migration harness -This folder is the operator-facing harness for the Rust migration test plan. It runs pytest normally, listens to test events in-process, and redraws a live matrix grouped by testing strategy and SDK-level function. +This local harness follows [the agreed structure](AGENTS.md). The root command selects strategies and combines their reports. Each strategy has an independent entry point -The matrix always has these SDK columns: - -- `ocr / aocr` -- `messages / amessages` -- `responses / aresponses` -- `count_tokens` -- `chat_completions / acompletion` -- `transcription / atranscription` - -The harness has four deliberately broad test-strategy folders: - -| Strategy | Folder | -| --- | --- | -| Public SDK parity over generated and recorded inputs | [`e2e_fuzz_tests/`](e2e_fuzz_tests/) | -| Focused tests of Rust-owned behavior | [`unit_tests_rust/`](unit_tests_rust/) | -| Isolated transform and Python-to-Rust helper coverage | [`validate_sub_methods/`](validate_sub_methods/) | -| Already-existing live-API SDK tests | [`existing_e2e_test_sdk/`](existing_e2e_test_sdk/) | - -## Run it - -From the repository root: - -```bash -poetry run python -m tests.rust-python-harness +```text +strategies/ + e2e_parity/runner.py + sdk/ocr/fixtures/ + sdk/messages/ + sdk/chat_completions/ + sdk/responses/ + gateway/ + existing_e2e_test_sdk/runner.py + trace_parity/runner.py + sdk/ + gateway/ + unit_tests/ + runner.py + mapping_validator.py + python_runner.py + rust_runner.py +shared/ + parity/ + tracing/ + reporting/ ``` -The default runs every configured test once and updates all matching cells in real time. Narrow a run by strategy, SDK function, or both: +## Run locally ```bash -poetry run python -m tests.rust-python-harness --strategy e2e_fuzz_tests -poetry run python -m tests.rust-python-harness --function messages -poetry run python -m tests.rust-python-harness --strategy validate_sub_methods --function ocr +uv run python -m tests.rust-python-harness --list +uv run python -m tests.rust-python-harness --function ocr --plain +uv run python -m tests.rust-python-harness --strategy e2e_parity --surface sdk --function ocr --plain +uv run python -m tests.rust-python-harness.strategies.e2e_parity.runner --function ocr --plain +uv run python -m tests.rust-python-harness.strategies.trace_parity.runner --plain +uv run python -m tests.rust-python-harness.strategies.unit_tests.runner --plain +uv run python -m tests.rust-python-harness.strategies.existing_e2e_test_sdk.runner --function transcription --plain ``` -For a guided run, use the interactive picker. It asks which strategy rows and SDK -function columns to include, then hands the terminal to the live dashboard. It never -captures keys while tests are running, so Ctrl-C and pytest debugging remain safe. +Use `--interactive` for strategy and function selection, `--pytest-arg=-x` to stop pytest on its first failure, and `--coverage` to write Python coverage under `target/rust-python-harness/`. The harness enables pytest namespace-package discovery only for its own invocations -```bash -poetry run python -m tests.rust-python-harness --interactive -``` +This harness has no CI execution. A configured test that fails or disappears makes the command fail. An unconfigured strategy cell remains planned and contributes no passing evidence. Interruptions and collection errors stop execution; ordinary test failures remain in the combined report while later strategies run -Useful operator options: +## Strategy responsibilities -```bash -# Inspect coverage and pytest selectors without running anything. -poetry run python -m tests.rust-python-harness --list +E2E parity compares SDK objects, exceptions, callbacks, streams, and provider requests. Gateway tests compare HTTP responses. Both surfaces use the same strategy runner and keep execution details and fixtures in their own folders. OCR has recorded sync/async SDK coverage; the existing Messages and Responses bridge checks remain partial -# Stable line-oriented output for CI logs or redirected output. -poetry run python -m tests.rust-python-harness --plain +Trace parity compares operation names through an explicit Python/Rust mapping, call counts, and required completion-before-start ordering with `shared/tracing/compare.py`. Surface tests supply captured operation intervals. No production trace instrumentation or trace case is configured yet -# Measure Python reference lines exercised by this parity run and build an HTML heatmap. -poetry run python -m tests.rust-python-harness --coverage +Unit testing combines test mapping validation, separate Python processes with Rust disabled and enabled, backend verification, result comparison, and native Cargo tests. Native tests stay beside their Rust implementation. Existing Python tests stay at their original paths. No complete Python/native unit mapping is configured yet, so these cells remain planned -# Forward pytest options. Use the equals form when the value begins with a dash. -poetry run python -m tests.rust-python-harness --pytest-arg=-x -``` +The existing E2E SDK strategy retains the live provider tests configured upstream. It runs OCR, Chat Completions, and Transcription checks from their existing paths and reports them separately from parity tests. These tests require provider credentials -The process returns pytest's exit code. A configured selector that collects no test is also a failure. A planned cell has no selector yet and does not fail the run. +## Configure cases -The dashboard adapts to narrow terminals, shows elapsed time and unique-test progress, -and prints the three slowest tests when the run ends. Each failure includes a focused -`poetry run pytest ... -q` command. Redirected output and CI automatically use the -line-oriented plain renderer; `--plain` lets you opt into it locally. - -The final screen includes a confidence score for every SDK section. It is the direct -ratio of required strategy rows with passing evidence, such as `1/3 = 33%`; High means -all required strategies passed, Medium means some passed, and Low means none passed. -This behavioral score is intentionally shown separately from Python and Rust LOC. - -Coverage reports are written outside the three strategy folders at -`target/rust-python-harness/`. Open `python-html/index.html` to inspect executed and -missing Python lines; `python.json` and `python.xml` are available for automation. -Coverage is finalized after pytest exits, because worker processes must flush their -data first. - -## Port coverage and confidence - -Treat these as separate signals instead of one ambiguous coverage percentage: - -| Signal | Tool | What it proves | -| --- | --- | --- | -| Python reference LOC | `coverage.py` / `pytest-cov` via `--coverage` | The mapped Python behavior ran | -| Rust port LOC | `cargo-llvm-cov` | The mapped Rust implementation ran | -| Parity contracts | This harness matrix | Python and Rust had the same observable behavior | - -`validate_sub_methods/` owns the future source-section inventory that maps a stable -Python qualified symbol to its Rust symbol. That inventory is the denominator for -per-function rollups; raw coverage for the entire LiteLLM repository would obscure -the port's real gaps. `unit_tests_rust/` owns direct `cargo-llvm-cov` runs, while -`e2e_fuzz_tests/` owns behavioral parity and fuzz-case counts. Keep Python, Rust, and -parity percentages visible side by side and label section confidence High only when -the mapped implementation exists, every required strategy passes, and both sides meet -their LOC thresholds. Generated Rust LCOV/HTML and the combined index also belong in -`target/rust-python-harness/`, not in a fourth strategy folder. - -## Read the matrix - -| Mark | Meaning | -| --- | --- | -| `✓` | All collected tests passed | -| `✗` | At least one test failed | -| `!` | Test setup or teardown failed | -| `↷` | All collected tests skipped | -| `?` | A configured selector did not collect a test | -| `—` | Strategy is planned but has no test yet | -| `n/a` | Strategy does not apply to this SDK function | -| `◐` | The configured tests cover only part of the TDD's parity contract | - -The initial end-to-end entries deliberately show `◐`: the repository has Rust bridge tests for OCR, Messages, and Responses websocket plumbing, but those are not yet frozen-Python-oracle comparisons. The remaining TDD cells stay visible as planned work instead of disappearing from a green summary. - -## Attach parity tests - -Each of the four folders contains a concise `README.md` and a `strategy.json`. Add a pytest file or node ID to the appropriate SDK function's `selectors` list: +Each strategy has a `strategy.json`. Its `functions` object defines SDK cases for OCR, Messages, Responses, Count Tokens, Chat Completions, and Transcription. E2E and trace manifests also accept a `gateway` object keyed by API name. A case has `coverage`, `selectors`, and an optional `note` ```json { - "coverage": "complete", - "selectors": [ - "tests/rust-python-harness/validate_sub_methods/test_messages.py" - ] + "coverage": "partial", + "selectors": ["tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py"] } ``` -Selectors use the same syntax as pytest. A file selector aggregates every test in the file; a node selector can target one test or parametrized family; a selector ending in `/` aggregates every test in that folder, recursively. The runner deduplicates selectors, so one test may intentionally prove more than one cell without executing twice. +Selectors use pytest file or node syntax. A selector ending in `/` includes tests recursively from that directory -Use these coverage values: +Use `planned` with no selectors until an executable contract exists, `partial` for incomplete coverage, `complete` for the full contract, and `not_applicable` when a strategy does not apply. The dashboard shows passing evidence separately from coverage completeness and LOC coverage -- `complete`: implements the full strategy contract for that SDK function. -- `partial`: useful coverage exists, but the TDD contract is not fully proven. -- `planned`: no runnable parity test exists yet. -- `not_applicable`: the strategy cannot apply, such as streaming for OCR. +Unit cases use `unit_suite` instead of `selectors`, pointing to a repository-relative JSON file with this shape: -Keep comparison mechanics in shared harness modules and provider/function facts in the owning strategy folder. A Python/Rust mismatch is a test failure; do not normalize away observable return types, exception classes, private response fields, chunk ordering, or callback payload differences merely to make a cell green. +```json +{ + "python_selectors": ["tests/test_api.py::test_decode"], + "cargo_manifest": "litellm-rust/Cargo.toml", + "cargo_package": "litellm-core", + "cargo_filter": "ocr::", + "backend": { + "environment_variable": "LITELLM_USE_RUST_OCR", + "probe": "tests.rust-python-harness.strategies.unit_tests.python_runner:ocr_backend" + }, + "mappings": [{"python": "tests/test_api.py::test_decode", "rust": "ocr::test_decode"}] +} +``` -## Architecture +Names match automatically when the collected Python and Rust test names agree. Explicit `mappings` handle different names, class names, and parametrized cases. Missing or ambiguous counterparts fail validation in either direction. The Cargo filter must select the same behavior as the Python selectors -- `catalog.py` validates and loads every strategy manifest. -- `models.py` owns typed strategy, case, coverage, and run-state models. -- `runner.py` maps live pytest events back to one or more matrix cells. -- `ui.py` renders the interactive Rich dashboard and a dependency-free plain fallback. -- `cli.py` handles filtering and preserves pytest exit semantics. +The backend probe returns `python` or `rust` and runs at startup and before every test call, after fixtures have run. The OCR probe verifies the dispatch flag and native extension availability. Surface tests must also assert that calls reach their intended implementation to catch per-call fallback. Python outcomes must agree, and failed runs remain failures even if both backends fail identically -The harness is driven from Python, matching the SDK surface and existing test tooling. Rust remains responsible for the implementation under comparison; the harness does not move provider semantics into the PyO3 bridge. +## OCR fixtures + +Fixtures, provider configuration, input strategies, and recording commands live in [the OCR package](strategies/e2e_parity/sdk/ocr/fixtures/README.md). Record with provider credentials: + +```bash +uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.record --examples 1000 +``` + +`LITELLM_OCR_FIXTURE_DIR` and `--fixture-dir` override the default directory. Shared recording, replay, comparison, streaming, and cassette persistence live in `shared/parity/` + +Run the harness's own checks locally: + +```bash +uv run pytest -o consider_namespace_packages=true tests/rust-python-harness/shared tests/rust-python-harness/strategies/unit_tests tests/test_rust_python_harness.py -q +``` + +Existing OCR parity gaps remain visible: invalid-model provider errors differ, Reducto lacks a native contract, and the expanded Azure corpus exposes duplicate Content-Type headers. Moving the harness does not change provider responses or weaken assertions diff --git a/tests/rust-python-harness/catalog.py b/tests/rust-python-harness/catalog.py index e23b9b125f0..f40fd5fc6b0 100644 --- a/tests/rust-python-harness/catalog.py +++ b/tests/rust-python-harness/catalog.py @@ -2,92 +2,74 @@ from __future__ import annotations import json from pathlib import Path -from typing import Any +from typing import Final -from .models import Coverage, HarnessCase, SDK_FUNCTIONS, Strategy +from pydantic import BaseModel, ConfigDict, ValidationError -STRATEGIES_ROOT = Path(__file__).parent +from .shared.reporting.models import Coverage, HarnessCase, SDK_FUNCTIONS, Strategy + +STRATEGIES_ROOT: Final = Path(__file__).parent / "strategies" -def _require_string(value: Any, field: str, source: Path) -> str: - if not isinstance(value, str) or not value.strip(): - raise ValueError(f"{source}: {field} must be a non-empty string") - return value +class CaseSpec(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + coverage: Coverage + selectors: tuple[str, ...] = () + note: str = "" + unit_suite: str | None = None + + +class StrategySpec(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + order: int + id: str + label: str + description: str + functions: dict[str, CaseSpec] + gateway: dict[str, CaseSpec] = {} def _load_strategy(source: Path) -> Strategy: - with source.open(encoding="utf-8") as stream: - data = json.load(stream) - - strategy_id = _require_string(data.get("id"), "id", source) - label = _require_string(data.get("label"), "label", source) - description = _require_string(data.get("description"), "description", source) - order = data.get("order") - if not isinstance(order, int): - raise ValueError(f"{source}: order must be an integer") - function_data = data.get("functions") - if not isinstance(function_data, dict): - raise ValueError(f"{source}: functions must be an object") - - missing = set(SDK_FUNCTIONS) - set(function_data) - extra = set(function_data) - set(SDK_FUNCTIONS) - if missing or extra: - raise ValueError( - f"{source}: functions must exactly match {SDK_FUNCTIONS}; missing={missing}, extra={extra}" + data: Final = StrategySpec.model_validate_json(source.read_text(encoding="utf-8")) + if set(data.functions) != set(SDK_FUNCTIONS): + raise ValueError(f"{source}: functions must exactly match {SDK_FUNCTIONS}") + cases: Final = tuple( + HarnessCase( + strategy_id=data.id, + strategy_label=data.label, + sdk_function=name, + coverage=case.coverage, + selectors=case.selectors, + note=case.note, + surface=surface, + unit_suite=case.unit_suite, ) - - cases: list[HarnessCase] = [] - for sdk_function in SDK_FUNCTIONS: - case_data = function_data[sdk_function] - if not isinstance(case_data, dict): - raise ValueError(f"{source}: functions.{sdk_function} must be an object") - try: - coverage = Coverage(case_data.get("coverage")) - except ValueError as exc: - raise ValueError(f"{source}: invalid coverage for {sdk_function}") from exc - selectors = case_data.get("selectors", []) - if not isinstance(selectors, list) or not all( - isinstance(item, str) and item for item in selectors - ): - raise ValueError( - f"{source}: selectors for {sdk_function} must be a list of strings" - ) - if coverage is Coverage.NOT_APPLICABLE and selectors: - raise ValueError( - f"{source}: not_applicable case {sdk_function} cannot have selectors" - ) - cases.append( - HarnessCase( - strategy_id=strategy_id, - strategy_label=label, - sdk_function=sdk_function, - coverage=coverage, - selectors=tuple(selectors), - note=str(case_data.get("note", "")), - ) - ) - - return Strategy( - order=order, - id=strategy_id, - label=label, - description=description, - directory=source.parent, - cases=tuple(cases), + for surface, functions in (("sdk", data.functions), ("gateway", data.gateway)) + for name in (SDK_FUNCTIONS if surface == "sdk" else functions) + for case in (functions[name],) ) + for case in cases: + if case.coverage in {Coverage.PLANNED, Coverage.NOT_APPLICABLE} and (case.selectors or case.unit_suite): + raise ValueError(f"{source}: {case.coverage.value} case {case.key} cannot configure tests") + if any(not selector.strip() for selector in case.selectors): + raise ValueError(f"{source}: empty selector in {case.key}") + if data.id == "unit_tests" and case.selectors: + raise ValueError(f"{source}: unit_tests must configure unit_suite instead of pytest selectors") + if data.id != "unit_tests" and case.unit_suite: + raise ValueError(f"{source}: unit_suite is only valid for unit_tests") + return Strategy(data.order, data.id, data.label, data.description, source.parent, cases) def load_catalog(root: Path = STRATEGIES_ROOT) -> tuple[Strategy, ...]: - sources = sorted(root.glob("*/strategy.json")) + sources: Final = tuple(sorted(root.glob("*/strategy.json"))) if not sources: raise ValueError(f"No strategy manifests found below {root}") - strategies = tuple( - sorted( - (_load_strategy(source) for source in sources), - key=lambda strategy: strategy.order, - ) - ) - ids = [strategy.id for strategy in strategies] - if len(ids) != len(set(ids)): + try: + strategies: Final = tuple(sorted((_load_strategy(source) for source in sources), key=lambda item: item.order)) + except (ValidationError, json.JSONDecodeError) as error: + raise ValueError(str(error)) from error + if len({strategy.id for strategy in strategies}) != len(strategies): raise ValueError(f"Duplicate strategy id in {root}") return strategies diff --git a/tests/rust-python-harness/cli.py b/tests/rust-python-harness/cli.py index c996b68846c..d266a12ce92 100644 --- a/tests/rust-python-harness/cli.py +++ b/tests/rust-python-harness/cli.py @@ -6,10 +6,14 @@ from collections.abc import Sequence from pathlib import Path from .catalog import load_catalog -from .models import SDK_FUNCTIONS, HarnessCase, Strategy -from .runner import run_pytest -from .ui import make_dashboard +from .shared.reporting.models import SDK_FUNCTIONS, HarnessCase, Strategy +from .shared.reporting.orchestration import StrategyRunner, run_strategies +from .shared.reporting.ui import make_dashboard +from .strategies.e2e_parity.runner import run as run_e2e +from .strategies.existing_e2e_test_sdk.runner import run as run_existing +from .strategies.trace_parity.runner import run as run_trace from .strategies.unit_tests.mapping_validator import FunctionReport, build_function_report +from .strategies.unit_tests.runner import run as run_units REPO_ROOT = Path(__file__).resolve().parents[2] COVERAGE_ROOT = REPO_ROOT / "target" / "rust-python-harness" @@ -44,6 +48,7 @@ def _parser() -> argparse.ArgumentParser: choices=SDK_FUNCTIONS, help="run only this SDK function", ) + parser.add_argument("--surface", choices=("sdk", "gateway"), help="run only this API surface") parser.add_argument( "--validate-ledger", action="store_true", @@ -135,9 +140,9 @@ def _print_catalog(strategies: Sequence[Strategy]) -> None: print(f"{strategy.id:20} {strategy.label}") for case in strategy.cases: selectors = ( - ", ".join(case.selectors) if case.selectors else "no test configured" + ", ".join(case.selectors) if case.selectors else case.unit_suite or "no test configured" ) - print(f" {case.sdk_function:12} {case.coverage.value:14} {selectors}") + print(f" {case.surface}/{case.sdk_function:12} {case.coverage.value:14} {selectors}") def _print_function_report(report: FunctionReport) -> None: @@ -172,7 +177,21 @@ def _validate_ledger(sdk_functions: set[str]) -> int: return 0 if all(report.is_clean for report in reports) else 1 -def main(argv: Sequence[str] | None = None) -> int: +def _resolve_runner(strategy_id: str) -> StrategyRunner: + match strategy_id: + case "e2e_parity": + return run_e2e + case "trace_parity": + return run_trace + case "unit_tests": + return run_units + case "existing_e2e_test_sdk": + return run_existing + case _: + raise ValueError(f"Unknown strategy: {strategy_id}") + + +def main(argv: Sequence[str] | None = None, *, strategy_id: str | None = None) -> int: args = _parser().parse_args(argv) if args.coverage and importlib.util.find_spec("pytest_cov") is None: _parser().error( @@ -181,7 +200,8 @@ def main(argv: Sequence[str] | None = None) -> int: ) if args.validate_ledger: return _validate_ledger(set(args.sdk_functions)) - strategies = load_catalog() + catalog = load_catalog() + strategies = tuple(strategy for strategy in catalog if strategy_id is None or strategy.id == strategy_id) if args.list: _print_catalog(strategies) return 0 @@ -194,7 +214,8 @@ def main(argv: Sequence[str] | None = None) -> int: sdk_functions = sdk_functions or picked_functions try: - cases = _select(strategies, strategy_ids, sdk_functions) + selected = _select(strategies, strategy_ids, sdk_functions) + cases = tuple(case for case in selected if args.surface is None or case.surface == args.surface) except ValueError as exc: _parser().error(str(exc)) selected_strategy_ids = {case.strategy_id for case in cases} @@ -210,11 +231,12 @@ def main(argv: Sequence[str] | None = None) -> int: if args.coverage: pytest_args.extend(_coverage_pytest_args()) with dashboard: - exit_code, run = run_pytest( + exit_code, run = run_strategies( cases=cases, repo_root=REPO_ROOT, on_update=dashboard.update, pytest_args=pytest_args, + resolve_runner=_resolve_runner, ) dashboard.finish(run, exit_code) if args.coverage and (COVERAGE_ROOT / "python.json").exists(): diff --git a/tests/rust-python-harness/e2e_fuzz_tests/README.md b/tests/rust-python-harness/e2e_fuzz_tests/README.md deleted file mode 100644 index 34b12050ff9..00000000000 --- a/tests/rust-python-harness/e2e_fuzz_tests/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# End-to-end fuzz tests - -Runs the same SDK call through the Python and Rust paths using generated inputs and recorded provider responses. It compares public results, streams, callbacks, and exceptions to catch behavior differences a unit test can miss. diff --git a/tests/rust-python-harness/e2e_fuzz_tests/strategy.json b/tests/rust-python-harness/e2e_fuzz_tests/strategy.json deleted file mode 100644 index d838486772d..00000000000 --- a/tests/rust-python-harness/e2e_fuzz_tests/strategy.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "order": 10, - "id": "e2e_fuzz_tests", - "label": "End-to-end fuzz tests", - "description": "Compare observable Python and Rust SDK behavior over generated and recorded inputs.", - "functions": { - "ocr": {"coverage": "partial", "selectors": ["tests/test_litellm/ocr/test_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}, - "messages": {"coverage": "partial", "selectors": ["tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}, - "responses": {"coverage": "partial", "selectors": ["tests/test_litellm/responses/test_rust_bridge_websocket.py"], "note": "Covers the websocket bridge; full responses parity is still being added."}, - "count_tokens": {"coverage": "planned", "selectors": [], "note": "No Rust count_tokens parity test is present yet."}, - "chat_completions": {"coverage": "partial", "selectors": ["tests/test_litellm/rust_bridge/test_chat_completions.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."}, - "transcription": {"coverage": "partial", "selectors": ["tests/test_litellm/test_audio_transcription_rust_bridge.py"], "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added."} - } -} diff --git a/tests/rust-python-harness/shared/parity/README.md b/tests/rust-python-harness/shared/parity/README.md new file mode 100644 index 00000000000..468dc531a99 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/README.md @@ -0,0 +1,91 @@ +# Implementation parity testing through the SDK interface + +> Given the same SDK call and identical provider behavior, do two implementations expose the same SDK contract? + +## What the harness compares + +- A fixture contains a LiteLLM SDK input and a recorded upstream provider response +- The same LiteLLM input is transformed by isolated baseline and candidate implementations +- The resulting provider requests must match in method, path, headers, and body, excluding runtime-specific HTTP metadata +- The recorded provider response is then replayed unchanged to both workers +- The harness compares the values returned through the Python SDK interface +- Non-streaming responses are compared directly, including their concrete return type and public model fields +- Streaming responses are consumed and compared chunk by chunk, including wrapper type, chunk type and order, termination, and public exception behavior +- Failed SDK calls are compared by exception class, stable message, status, code, model, provider, and parameter fields +- Traceback paths and line numbers are excluded because they are runtime-specific +- Route-specific comparators and chunk normalizers handle differences in each public SDK contract + +## Process isolation + +- SDK object and stream parity runs both implementations sequentially in the same process so tests can retain returned objects +- Every test saves and restores the original bridge state +- A small subprocess smoke test verifies environment-based startup configuration and detects fallback to the Python HTTP implementation + +## Streaming execution + +The invocation callback passed to `run_in_process` must consume the stream before returning its `StreamOutcome`. +Use `consume_sync_stream` inside that callback, or await `consume_async_stream` inside the callback passed to +`run_in_process_async`. Provider requests are collected only after the callback completes. Streaming is explicit: +an iterable return value alone does not select stream consumption + +The consumers retain the wrapper type, iteration capabilities, chunk types and order, and any partial output before +an error. Errors retain their creation or iteration phase and the full public `SDKError` fields, with traceback text +removed. `capture_sync_stream` and `capture_async_stream` consume through the same helpers and then serialize the +outcome for subprocess reports. A serialization failure raises as a harness failure rather than becoming an SDK error + +Response models and stream chunks share a recursive comparator. It compares concrete model, container, and scalar +types, public fields and extras, and exact values while ignoring Pydantic private attributes at every nesting level. +An API may supply an explicit chunk normalizer for its public contract + +Shared tests exercise a local SSE provider through recording, VCR cassette storage, replay, and typed event comparison +in sync and async modes. They cover fragmented events, split UTF-8 characters, CRLF framing, coalesced events, and +application errors within a normally completed HTTP stream. HTTP byte boundaries and decoded SDK event boundaries +are checked separately + +OCR remains the only integrated LiteLLM route. These tests validate shared streaming machinery, not another route's +SDK parity. Connection interruption, early cancellation, and lifecycle timeout enforcement remain outside this coverage + +## Hypothesis and property-based testing + +- Hypothesis is a Python library for property-based testing +- Example-based tests use inputs selected by the test author +- Property-based tests define strategies for valid inputs and properties that must hold for every generated example +- Hypothesis generates combinations from those strategies and normally shrinks a failing example to a smaller reproducible case +- In this harness, Hypothesis is used only during fixture generation to expand the LiteLLM input corpus +- Each API owns the strategies that vary its supported inputs +- Fixture generation is deterministic, and each generated input is recorded with the raw provider response it received +- The parity tests use committed fixtures and do not call the provider or generate new Hypothesis examples +- Provider responses are replayed unchanged, so the parity test does not fuzz or validate provider behavior +- Because Hypothesis does not run the parity assertion directly, parity failures are not automatically shrunk + +## API-owned fixtures + +The shared package owns recording, replay, persistence, execution, comparison, and route-neutral media constructors. +Each API package owns its input models, explicit strategies, provider targets, route-specific assets, fixture directory, +and regeneration command. See the API package documentation for its configured contracts and recording command + +## VCR cassettes + +Fixtures use VCR's YAML `version: 1` format with ordered request/response `interactions`. VCR handles text and binary +body serialization. Each cassette also contains `recorded_at`, `ttl_seconds: 0` (committed fixtures never expire), and +`x-litellm` metadata holding the SDK input and request provenance. Streaming responses carry +`x-litellm-chunk-lengths` so local replay preserves the original byte boundaries + +The recording server captures requests before forwarding their responses. Saved requests use the stable +`http://parity-provider.invalid` origin and strip authentication headers and credential query parameters. The upstream +request keeps its credentials. Provider response bytes and non-success statuses are preserved + +Standard VCR can load these files and replay their interactions. Parity tests keep using the local HTTP server because +Rust HTTP calls do not pass through VCR's Python patches. The harness still compares the two implementations' requests +against each other; the saved request is available for inspection and VCR playback, not a new parity assertion + +Refresh parity cassettes through the API's recording command. Generic VCR writers do not preserve the SDK metadata + +Legacy JSON fixtures remain readable. Migrated cassettes mark reconstructed requests as `python_replay`; fresh +recordings use `recorded`. The metadata extensions follow the filesystem cassette layout proposed in +[PR #39338](https://github.com/BerriAI/litellm/pull/39338), without depending on its unmerged persistence backend + +## References + +- [Hypothesis documentation](https://hypothesis.readthedocs.io/en/latest/) +- [Hypothesis documentation source](https://github.com/HypothesisWorks/hypothesis/tree/master/hypothesis/docs) diff --git a/tests/rust-python-harness/shared/parity/__init__.py b/tests/rust-python-harness/shared/parity/__init__.py index e69de29bb2d..f18197acd2a 100644 --- a/tests/rust-python-harness/shared/parity/__init__.py +++ b/tests/rust-python-harness/shared/parity/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.register_assert_rewrite("tests.rust-python-harness.shared.parity.compare") diff --git a/tests/rust-python-harness/shared/parity/compare.py b/tests/rust-python-harness/shared/parity/compare.py new file mode 100644 index 00000000000..adf85e5c8d7 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/compare.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from collections.abc import Mapping, Sequence +from typing import Final, cast + +from pydantic import BaseModel + +from .models import CapturedRequest, Execution + + +def validate_harness(baseline: Execution, candidate: Execution, baseline_user_agent: str) -> None: + for request in baseline.requests: + if request.user_agent != baseline_user_agent: + raise AssertionError( + f"baseline provider request did not carry sentinel user-agent {baseline_user_agent!r}: " + f"{request.user_agent!r}" + ) + for request in candidate.requests: + if request.user_agent == baseline_user_agent: + raise AssertionError("candidate route fell back to the baseline HTTP implementation") + + +def _request_after_transformation(request: CapturedRequest) -> CapturedRequest: + return request.model_copy(update={"user_agent": None}) + + +def assert_request_parity(baseline: tuple[CapturedRequest, ...], candidate: tuple[CapturedRequest, ...]) -> None: + baseline_requests: Final = tuple(_request_after_transformation(request) for request in baseline) + candidate_requests: Final = tuple(_request_after_transformation(request) for request in candidate) + assert_value_parity(baseline_requests, candidate_requests) + + +def _public_model_values(model: BaseModel) -> dict[str, object]: + fields: Final = (*type(model).model_fields, *type(model).model_computed_fields) + extras: Final = cast(Mapping[str, object], model.model_extra or {}) + return { + **{name: cast(object, getattr(model, name)) for name in fields if not name.startswith("_")}, + **{name: value for name, value in extras.items() if not name.startswith("_")}, + } + + +def assert_model_parity(baseline: BaseModel, candidate: BaseModel) -> None: + assert_value_parity(baseline, candidate) + + +def assert_value_parity(baseline: object, candidate: object, *, path: str = "$") -> None: + assert type(baseline) is type(candidate), f"type mismatch at {path}: {type(baseline)} != {type(candidate)}" + if isinstance(baseline, BaseModel) and isinstance(candidate, BaseModel): + assert_value_parity(_public_model_values(baseline), _public_model_values(candidate), path=path) + return + if isinstance(baseline, Mapping) and isinstance(candidate, Mapping): + baseline_mapping: Final = cast(Mapping[object, object], baseline) + candidate_mapping: Final = cast(Mapping[object, object], candidate) + assert frozenset((type(key), key) for key in baseline_mapping) == frozenset( + (type(key), key) for key in candidate_mapping + ), f"mapping keys differ at {path}" + for key in baseline_mapping: + assert_value_parity(baseline_mapping[key], candidate_mapping[key], path=f"{path}.{key}") + return + if ( + isinstance(baseline, Sequence) + and not isinstance(baseline, (str, bytes)) + and isinstance(candidate, Sequence) + and not isinstance(candidate, (str, bytes)) + ): + baseline_sequence: Final = cast(Sequence[object], baseline) + candidate_sequence: Final = cast(Sequence[object], candidate) + assert len(baseline_sequence) == len(candidate_sequence), f"sequence lengths differ at {path}" + for index, (baseline_item, candidate_item) in enumerate( + zip(baseline_sequence, candidate_sequence, strict=True) + ): + assert_value_parity(baseline_item, candidate_item, path=f"{path}[{index}]") + return + assert baseline == candidate, f"value mismatch at {path}: {baseline!r} != {candidate!r}" + + +def assert_parity(baseline: Execution, candidate: Execution, baseline_user_agent: str) -> None: + validate_harness(baseline, candidate, baseline_user_agent) + assert_request_parity(baseline.requests, candidate.requests) + assert_value_parity(baseline.report, candidate.report) diff --git a/tests/rust-python-harness/shared/parity/fixture_models.py b/tests/rust-python-harness/shared/parity/fixture_models.py new file mode 100644 index 00000000000..64023061330 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixture_models.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import ClassVar, Final, Generic, Literal, TypeVar, cast + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, model_validator + +from .recorded_http import RecordedResponse + +JsonObject = dict[str, JsonValue] + + +class FixtureModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True, serialize_by_alias=True) + + +class SdkInputBase(FixtureModel): + fixture_only_fields: ClassVar[tuple[str, ...]] = () + + def as_sdk_kwargs(self) -> dict[str, object]: + return cast( + dict[str, object], + self.model_dump( + mode="python", + exclude_unset=True, + exclude=set(self.fixture_only_fields), + ), + ) + + def canonical_input(self) -> dict[str, object]: + dumped: Final = cast(dict[str, object], self.model_dump(mode="json", exclude_unset=True)) + fixture_fields: Final = {field: getattr(self, field) for field in self.fixture_only_fields} + return {**fixture_fields, **dumped} + + +class JsonSchemaDefinition(FixtureModel): + name: str + description: str | None = None + schema_definition: JsonObject = Field(alias="schema") + strict: bool = False + + +class JsonSchemaResponseFormat(FixtureModel): + type: Literal["json_schema"] + json_schema: JsonSchemaDefinition + + +InputT = TypeVar("InputT", bound=SdkInputBase) + + +class ParityCase(FixtureModel, Generic[InputT]): + litellm_input: InputT + provider_responses: tuple[RecordedResponse, ...] + + @model_validator(mode="before") + @classmethod + def load_legacy_single_response(cls, value: object) -> object: + if not isinstance(value, Mapping): + return value + migrated: Final = dict(cast(Mapping[str, object], value)) + provider_response: Final = migrated.pop("provider_response", None) + if "provider_responses" not in migrated and provider_response is not None: + migrated["provider_responses"] = (provider_response,) + return migrated diff --git a/tests/rust-python-harness/shared/parity/fixtures/__init__.py b/tests/rust-python-harness/shared/parity/fixtures/__init__.py new file mode 100644 index 00000000000..9d48db4f9f8 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/tests/rust-python-harness/shared/parity/fixtures/cassette.py b/tests/rust-python-harness/shared/parity/fixtures/cassette.py new file mode 100644 index 00000000000..03a5fc9f416 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/cassette.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +from collections.abc import Mapping +from datetime import datetime +from itertools import accumulate +from typing import Final, Literal + +from pydantic import AwareDatetime, BaseModel, ConfigDict, Field, TypeAdapter +from vcr.serialize import serialize +from vcr.serializers import yamlserializer + +from .recording import RecordedInteraction +from ..recorded_http import ( + HttpHeader, + RecordedHttpResponse, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) + +_OBJECT: Final = TypeAdapter(dict[str, object]) + + +class _CassetteModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid", populate_by_name=True) + + +class _Body(_CassetteModel): + string: str | bytes + + def as_bytes(self) -> bytes: + return self.string.encode("utf-8") if isinstance(self.string, str) else self.string + + +class _Status(_CassetteModel): + code: int + message: str + + +class _Request(_CassetteModel): + method: str + uri: str + body: str | bytes | None + headers: dict[str, tuple[str, ...]] + + +class _Response(_CassetteModel): + status: _Status + headers: dict[str, tuple[str, ...]] + body: _Body + chunk_lengths: tuple[int, ...] | None = Field(default=None, alias="x-litellm-chunk-lengths") + + def recorded_response(self) -> RecordedResponse: + headers: Final = tuple( + HttpHeader(name=name, value=value) for name, values in self.headers.items() for value in values + ) + body: Final = self.body.as_bytes() + if self.chunk_lengths is None: + return RecordedHttpResponse.from_bytes(self.status.code, headers, body) + if any(length < 0 for length in self.chunk_lengths) or sum(self.chunk_lengths) != len(body): + raise ValueError("cassette stream chunk lengths do not match the response body") + offsets: Final = tuple(accumulate(self.chunk_lengths, initial=0)) + return RecordedHttpStreamResponse( + kind="http_stream", + status_code=self.status.code, + headers=headers, + chunks=tuple(RecordedStreamChunk.from_bytes(body[start:end]) for start, end in zip(offsets, offsets[1:])), + ) + + +class _Interaction(_CassetteModel): + request: _Request + response: _Response + + +class _ParityMetadata(_CassetteModel): + schema_version: Literal[1] + request_source: Literal["recorded", "python_replay"] + case: dict[str, object] + + +class ParityCassette(_CassetteModel): + version: Literal[1] + recorded_at: AwareDatetime + ttl_seconds: Literal[0] + interactions: tuple[_Interaction, ...] + parity: _ParityMetadata = Field(alias="x-litellm") + + def case_data(self) -> dict[str, object]: + return { + **self.parity.case, + "provider_responses": tuple(item.response.recorded_response() for item in self.interactions), + } + + +def _response_dict(response: RecordedResponse) -> dict[str, object]: + headers: Final = { + name: [header.value for header in response.headers if header.name == name] + for name in dict.fromkeys(header.name for header in response.headers) + } + chunks: Final = ( + tuple(chunk.data_bytes() for chunk in response.chunks) + if isinstance(response, RecordedHttpStreamResponse) + else None + ) + body: Final = response.body_bytes() if isinstance(response, RecordedHttpResponse) else b"".join(chunks or ()) + return { + "status": {"code": response.status_code, "message": ""}, + "headers": headers, + "body": {"string": body}, + **({"x-litellm-chunk-lengths": list(map(len, chunks))} if chunks is not None else {}), + } + + +def serialize_cassette( + case: Mapping[str, object], + interactions: tuple[RecordedInteraction, ...], + recorded_at: datetime, + request_source: Literal["recorded", "python_replay"], +) -> str: + normalized: Final = _OBJECT.validate_python( + yamlserializer.deserialize( + serialize( + { + "requests": [item.request for item in interactions], + "responses": [_response_dict(item.response) for item in interactions], + }, + yamlserializer, + ) + ) + ) + payload: Final = { + **normalized, + "recorded_at": recorded_at.isoformat(), + "ttl_seconds": 0, + "x-litellm": { + "schema_version": 1, + "request_source": request_source, + "case": {key: value for key, value in case.items() if key != "provider_responses"}, + }, + } + ParityCassette.model_validate(payload).case_data() + return str(yamlserializer.serialize(payload)) + + +def deserialize_cassette(contents: str) -> ParityCassette: + return ParityCassette.model_validate(yamlserializer.deserialize(contents)) diff --git a/tests/rust-python-harness/shared/parity/fixtures/cli.py b/tests/rust-python-harness/shared/parity/fixtures/cli.py new file mode 100644 index 00000000000..b6a2db1c395 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/cli.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import argparse +from collections.abc import Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + + +@dataclass(frozen=True, slots=True) +class RecordingArgs: + concurrency: int + examples: int + fixture_dir: Path | None + + +def _positive_int(value: str) -> int: + parsed: Final = int(value) + if parsed < 1: + raise argparse.ArgumentTypeError("must be at least 1") + return parsed + + +def parse_recording_args(argv: Sequence[str] | None = None) -> RecordingArgs: + parser: Final = argparse.ArgumentParser() + parser.add_argument("--concurrency", type=_positive_int, default=4) + parser.add_argument("--examples", type=_positive_int, default=4) + parser.add_argument("--fixture-dir", type=Path) + namespace: Final = parser.parse_args(argv) + return RecordingArgs( + concurrency=cast(int, namespace.concurrency), + examples=cast(int, namespace.examples), + fixture_dir=cast(Path | None, namespace.fixture_dir), + ) diff --git a/tests/rust-python-harness/shared/parity/fixtures/inputs.py b/tests/rust-python-harness/shared/parity/fixtures/inputs.py new file mode 100644 index 00000000000..bbca19d0104 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/inputs.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +import queue +from typing import Final, TypeVar + +from hypothesis import given, settings +from hypothesis.strategies import SearchStrategy + +InputT = TypeVar("InputT") + + +def generate_case_inputs(strategy: SearchStrategy[InputT], examples: int) -> tuple[InputT, ...]: + generated: Final[queue.SimpleQueue[InputT | None]] = queue.SimpleQueue() + + @settings(max_examples=examples, deadline=None, derandomize=True) + @given(case_input=strategy) + def generate_case(case_input: InputT) -> None: + generated.put(case_input) + + generate_case() + generated.put(None) + return tuple(iter(generated.get, None)) diff --git a/tests/rust-python-harness/shared/parity/fixtures/media.py b/tests/rust-python-harness/shared/parity/fixtures/media.py new file mode 100644 index 00000000000..d1ea2901f5d --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/media.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import base64 +from functools import cache +from io import BytesIO +from typing import Final +from urllib.parse import quote + +from PIL import Image, ImageDraw +from reportlab.graphics.barcode import code128 # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib import colors # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib.pagesizes import letter # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib.utils import ImageReader # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.pdfgen import canvas # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs + + +def dummy_image_url(text: str, font_size: int, width: int = 800, height: int = 300) -> str: + return f"https://dummyjson.com/image/{width}x{height}/ffffff/000000?text={quote(text)}&fontSize={font_size}" + + +_GLYPHS: Final = { + "D": ("11110", "10001", "10001", "10001", "10001", "10001", "11110"), + "O": ("01110", "10001", "10001", "10001", "10001", "10001", "01110"), + "C": ("01111", "10000", "10000", "10000", "10000", "10000", "01111"), + "1": ("00100", "01100", "00100", "00100", "00100", "00100", "01110"), + "2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"), + "3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"), +} + + +@cache +def structured_image_bytes() -> bytes: + image: Final = Image.new("RGB", (320, 80), "white") + draw: Final = ImageDraw.Draw(image) + scale: Final = 8 + cursor_x = 24 + for character in "DOC 123": + if character == " ": + cursor_x += scale * 3 + continue + for glyph_y, row in enumerate(_GLYPHS[character]): + for glyph_x, filled in enumerate(row): + if filled == "1": + x = cursor_x + glyph_x * scale + y = 12 + glyph_y * scale + draw.rectangle((x, y, x + scale - 1, y + scale - 1), fill="black") + cursor_x += scale * 6 + output: Final = BytesIO() + image.save(output, format="PNG") + return output.getvalue() + + +@cache +def structured_image_data_uri() -> str: + encoded: Final = base64.b64encode(structured_image_bytes()).decode("ascii") + return f"data:image/png;base64,{encoded}" + + +def _draw_header(pdf: canvas.Canvas, title: str, page_number: int) -> None: + pdf.setFillColor(colors.black) + pdf.setFont("Helvetica", 11) + pdf.drawString(45, 770, "Quarterly Operations Report") + pdf.setFont("Helvetica-Bold", 16) + pdf.drawString(45, 745, title) + pdf.setFont("Helvetica", 9) + pdf.drawString(45, 30, f"Confidential | Page {page_number} of 5") + + +def _draw_body(pdf: canvas.Canvas, page_number: int) -> None: + pdf.setFont("Helvetica", 10) + for line_number in range(1, 9): + pdf.drawString( + 45, + 500 - (line_number * 28), + f"Section {page_number}.{line_number}: Invoice totals, regional revenue, and reconciliation notes.", + ) + + +def _diagram_image(width: int, height: int, accent: tuple[int, int, int]) -> Image.Image: + image: Final = Image.new("RGB", (width, height), (242, 246, 252)) + draw: Final = ImageDraw.Draw(image) + for coordinate in range(0, max(width, height), 40): + draw.line((coordinate, 0, coordinate, height), fill=(32, 32, 32), width=3) + draw.line((0, coordinate, width, coordinate), fill=(32, 32, 32), width=3) + draw.line((0, 0, width, height), fill=accent, width=8) + draw.line((width, 0, 0, height), fill=accent, width=8) + draw.rectangle((width // 4, height // 4, width * 3 // 4, height * 3 // 4), outline=accent, width=6) + return image + + +def _draw_embedded_images(pdf: canvas.Canvas) -> None: + images: Final = ( + (_diagram_image(320, 320, (51, 115, 217)), 455, 655, 70, 70), + (_diagram_image(360, 320, (38, 151, 92)), 455, 565, 70, 62), + (_diagram_image(120, 120, (219, 68, 55)), 455, 500, 45, 45), + ) + for image, x, y, width, height in images: + pdf.drawImage( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + ImageReader(image), x, y, width=width, height=height, mask="auto" + ) + + +def _draw_table_page(pdf: canvas.Canvas) -> None: + columns: Final = (45, 245, 405, 565) + tables: Final = ( + ( + (730, 695, 660, 625), + ( + ("Item", "Quantity", "Amount", 707), + ("Document analysis", "2", "120.00", 672), + ("Document verification", "1", "80.00", 637), + ), + ), + ( + (600, 565, 530, 495), + ( + ("Item continued", "Quantity", "Amount", 577), + ("Fixture validation", "3", "45.00", 542), + ("Provider review", "1", "25.00", 507), + ), + ), + ) + for rows, values in tables: + for x in columns: + pdf.line(x, rows[-1], x, rows[0]) + for y in rows: + pdf.line(45, y, 565, y) + for item, quantity, amount, y in values: + pdf.drawString(55, y, item) + pdf.drawString(255, y, quantity) + pdf.drawString(415, y, amount) + + +def _draw_chart_page(pdf: canvas.Canvas) -> None: + bars: Final = ((70, 70), (170, 115), (270, 90), (370, 130)) + pdf.setFillColor(colors.HexColor("#3373D9")) + for x, height in bars: + pdf.rect(x, 610, 65, height, fill=1, stroke=0) + pdf.setFillColor(colors.black) + for quarter, x in zip(("Q1", "Q2", "Q3", "Q4"), (90, 190, 290, 390), strict=True): + pdf.drawString(x, 590, quarter) + pdf.drawString(45, 550, "Formula: gross margin = (revenue - cost) / revenue") + _draw_embedded_images(pdf) + + +def _draw_metadata_page(pdf: canvas.Canvas) -> None: + pdf.setFont("Helvetica", 12) + pdf.drawString(45, 700, "Invoice Number: INV-2048") + pdf.drawString(45, 675, "Purchase Order: PO-4096") + pdf.setFillColor(colors.HexColor("#F2E65A")) + pdf.rect(40, 555, 500, 24, fill=1, stroke=0) + pdf.setFillColor(colors.black) + pdf.drawString(45, 560, "Highlighted total requiring review") + pdf.drawString(45, 530, "Reviewer comment: verify the highlighted total before approval") + pdf.setFillColor(colors.red) + pdf.drawString(45, 495, "Revised total: 245.00") + pdf.line(45, 501, 150, 501) + pdf.setFillColor(colors.black) + pdf.linkURL( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "https://example.com/invoices/INV-2048", (45, 575, 300, 590), relative=0 + ) + pdf.highlightAnnotation( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "Total highlighted for review", + Rect=(40, 555, 540, 579), + QuadPoints=(40, 579, 540, 579, 40, 555, 540, 555), + ) + pdf.textAnnotation( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "Verify the highlighted total", Rect=(520, 525, 540, 545) + ) + pdf.drawString(45, 575, "https://example.com/invoices/INV-2048") + barcode: Final = code128.Code128("5901234123457", barHeight=70, barWidth=1.2) + barcode.drawOn(pdf, 90, 130) + + +def _draw_signature_page(pdf: canvas.Canvas) -> None: + pdf.saveState() + pdf.setFillColor(colors.lightgrey) + pdf.setFont("Helvetica-Bold", 54) + pdf.translate(110, 390) + pdf.rotate(25) + pdf.drawString(0, 0, "DRAFT") + pdf.restoreState() + pdf.setFillColor(colors.black) + pdf.setFont("Helvetica", 12) + pdf.drawString(45, 635, "Approved by: Jordan Lee") + pdf.line(45, 610, 310, 610) + pdf.bezier(55, 595, 75, 625, 112, 602, 155, 600) + pdf.drawString(45, 580, "Signature") + + +def _draw_appendix_page(pdf: canvas.Canvas) -> None: + pdf.setFont("Helvetica-Bold", 14) + pdf.drawString(45, 700, "1. Scope") + pdf.drawString(45, 650, "2. Findings") + pdf.drawString(45, 600, "3. Recommendations") + + +@cache +def structured_pdf_bytes() -> bytes: + output: Final = BytesIO() + pdf: Final = canvas.Canvas(output, pagesize=letter, pageCompression=0, invariant=1) + pdf.setTitle("Quarterly Operations Report") + pdf.setAuthor("LiteLLM parity fixture generator") + pdf.setSubject("Semantic document coverage for tables, figures, annotations, and metadata") + pdf.setKeywords("document, invoice, table, figure, annotation") + pages: Final = ( + ("Invoice Summary and Line Items", _draw_table_page), + ("Revenue Chart and Formula Review", _draw_chart_page), + ("Key Values, Link, Highlight, and Comment", _draw_metadata_page), + ("Approval Signature and Watermark", _draw_signature_page), + ("Appendix with Section Boundaries", _draw_appendix_page), + ) + for page_number, (title, draw_page) in enumerate(pages, start=1): + _draw_header(pdf, title, page_number) + draw_page(pdf) + _draw_body(pdf, page_number) + pdf.showPage() + pdf.save() + return output.getvalue() + + +@cache +def structured_pdf_data_uri() -> str: + encoded: Final = base64.b64encode(structured_pdf_bytes()).decode("ascii") + return f"data:application/pdf;base64,{encoded}" diff --git a/tests/rust-python-harness/shared/parity/fixtures/pipeline.py b/tests/rust-python-harness/shared/parity/fixtures/pipeline.py new file mode 100644 index 00000000000..d3adc45b22a --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/pipeline.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import logging +from concurrent.futures import Future, ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Final, Generic, Literal, Protocol, TypeVar + +from hypothesis.strategies import SearchStrategy +from pydantic import BaseModel + +from .inputs import generate_case_inputs +from .recording import UpstreamEndpoint, record_upstream_interactions +from .store import ( + FixtureInput, + canonical_json, + fixture_cache_key, + fixture_id, + fixture_path, + load_fixture, + save_fixture, +) + +LOGGER: Final = logging.getLogger(__name__) +InputT = TypeVar("InputT", bound=FixtureInput) +InputT_contra = TypeVar("InputT_contra", bound=FixtureInput, contravariant=True) +CaseT = TypeVar("CaseT", bound=BaseModel) + + +class RecordingInvocation(Protocol[InputT_contra]): + def execute(self, provider_url: str, case_input: InputT_contra) -> None: ... + + +@dataclass(frozen=True, slots=True) +class RecordingTarget(Generic[InputT]): + name: str + upstream: UpstreamEndpoint + strategy: SearchStrategy[InputT] + invocation: RecordingInvocation[InputT] = field(repr=False) + required_inputs: tuple[InputT, ...] = () + + +@dataclass(frozen=True, slots=True) +class RecordingJob(Generic[InputT]): + target_name: str + directory: Path + upstream: UpstreamEndpoint + case_input: InputT + invocation: RecordingInvocation[InputT] = field(repr=False) + + @property + def case_id(self) -> str: + return fixture_id(self.case_input, self.target_name) + + +@dataclass(frozen=True, slots=True) +class RecordedFixture: + target_name: str + case_id: str + path: Path + kind: Literal["recorded"] = field(default="recorded", init=False) + + +@dataclass(frozen=True, slots=True) +class CachedFixture: + target_name: str + case_id: str + path: Path + kind: Literal["cached"] = field(default="cached", init=False) + + +@dataclass(frozen=True, slots=True) +class FailedFixture: + target_name: str + case_id: str + error: Exception = field(repr=False) + kind: Literal["failed"] = field(default="failed", init=False) + + +RecordingOutcome = RecordedFixture | CachedFixture | FailedFixture + + +@dataclass(frozen=True, slots=True) +class RecordingSummary: + recorded: tuple[RecordedFixture, ...] + cached: tuple[CachedFixture, ...] + failed: tuple[FailedFixture, ...] + + @property + def exit_code(self) -> int: + return 1 if self.failed else 0 + + +def _unique_inputs(target: RecordingTarget[InputT], examples: int) -> tuple[InputT, ...]: + generated_inputs: Final = generate_case_inputs(target.strategy, examples) + case_inputs: Final = (*target.required_inputs, *generated_inputs) + return tuple({canonical_json(fixture_cache_key(case_input)): case_input for case_input in case_inputs}.values()) + + +def build_recording_jobs( + targets: tuple[RecordingTarget[InputT], ...], + root: Path, + examples: int, +) -> tuple[RecordingJob[InputT], ...]: + if examples < 1: + raise ValueError("examples must be at least 1") + return tuple( + RecordingJob( + target_name=target.name, + directory=root / target.name, + upstream=target.upstream, + case_input=case_input, + invocation=target.invocation, + ) + for target in targets + for case_input in _unique_inputs(target, examples) + ) + + +def _record_job(job: RecordingJob[InputT], case_type: type[CaseT]) -> RecordedFixture | CachedFixture: + cached: Final = load_fixture(job.directory, job.case_input, case_type) + if cached is not None: + path: Final = fixture_path(job.directory, job.case_input) + return CachedFixture( + target_name=job.target_name, + case_id=job.case_id, + path=path if path.is_file() else path.with_suffix(".json"), + ) + interactions: Final = record_upstream_interactions( + job.upstream, + job.case_input, + job.invocation.execute, + ) + case: Final = case_type.model_validate( + { + "litellm_input": job.case_input, + "provider_responses": tuple(item.response for item in interactions), + } + ) + saved_path: Final = save_fixture(job.directory, job.case_input, case, interactions) + return RecordedFixture(target_name=job.target_name, case_id=job.case_id, path=saved_path) + + +def _completed_outcome( + completed: int, + total: int, + job: RecordingJob[InputT], + future: Future[RecordedFixture | CachedFixture], +) -> RecordingOutcome: + try: + outcome: Final = future.result() + except Exception as error: + failed: Final = FailedFixture(target_name=job.target_name, case_id=job.case_id, error=error) + LOGGER.error( + "[%d/%d] failed %s %s: %s", + completed, + total, + failed.target_name, + failed.case_id, + type(error).__name__, + ) + return failed + LOGGER.info("[%d/%d] %s %s %s", completed, total, outcome.kind, outcome.target_name, outcome.case_id) + return outcome + + +def record_fixtures( + targets: tuple[RecordingTarget[InputT], ...], + root: Path, + examples: int, + concurrency: int, + case_type: type[CaseT], +) -> RecordingSummary: + if concurrency < 1: + raise ValueError("concurrency must be at least 1") + jobs: Final = build_recording_jobs(targets, root, examples) + total: Final = len(jobs) + LOGGER.info("Recording %d fixtures across %d targets with concurrency %d", total, len(targets), concurrency) + with ThreadPoolExecutor(max_workers=concurrency) as executor: + future_jobs: Final = MappingProxyType({executor.submit(_record_job, job, case_type): job for job in jobs}) + outcomes: Final = tuple( + _completed_outcome(completed, total, future_jobs[future], future) + for completed, future in enumerate(as_completed(future_jobs), start=1) + ) + summary: Final = RecordingSummary( + recorded=tuple(outcome for outcome in outcomes if isinstance(outcome, RecordedFixture)), + cached=tuple(outcome for outcome in outcomes if isinstance(outcome, CachedFixture)), + failed=tuple(outcome for outcome in outcomes if isinstance(outcome, FailedFixture)), + ) + LOGGER.info( + "Finished %d fixtures: %d recorded, %d cached, %d failed", + total, + len(summary.recorded), + len(summary.cached), + len(summary.failed), + ) + return summary diff --git a/tests/rust-python-harness/shared/parity/fixtures/pytest_support.py b/tests/rust-python-harness/shared/parity/fixtures/pytest_support.py new file mode 100644 index 00000000000..04c097f318c --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/pytest_support.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import os +from collections.abc import Callable +from pathlib import Path +from typing import Final, TypeVar + +import pytest +from pydantic import BaseModel, ValidationError + +from .store import recorded_fixtures + +CaseT = TypeVar("CaseT", bound=BaseModel) + + +def parametrize_recorded_fixtures( + metafunc: pytest.Metafunc, + *, + fixture_name: str, + case_type: type[CaseT], + env_var: str, + default_directory: Path, + regeneration_command: str, + id_builder: Callable[[CaseT], str], + marks_builder: Callable[[CaseT], tuple[pytest.MarkDecorator, ...]] | None = None, +) -> None: + if fixture_name not in metafunc.fixturenames: + return + configured: Final = os.environ.get(env_var) + if configured == "": + raise pytest.UsageError(f"{env_var} is set but empty") + directory: Final = Path(configured).expanduser() if configured is not None else default_directory + try: + fixtures: Final = recorded_fixtures(directory, case_type) + except (ValidationError, ValueError) as error: + raise pytest.UsageError( + f"Invalid parity fixture bundle at {directory}. " + "Each fixture must use the current versioned envelope. " + f"Record fresh fixtures in an empty directory with: `{regeneration_command}`. " + f"Validation details: {error}" + ) from error + if fixtures: + metafunc.parametrize( + fixture_name, + tuple( + pytest.param( + fixture, + id=id_builder(fixture), + marks=marks_builder(fixture) if marks_builder is not None else (), + ) + for fixture in fixtures + ), + ) + return + if configured is not None: + raise pytest.UsageError(f"no recorded fixtures in {directory}") + metafunc.parametrize( + fixture_name, + ( + pytest.param( + None, + marks=pytest.mark.skip(reason=f"no recorded fixtures in {directory}"), + id="no-recorded-fixtures", + ), + ), + ) diff --git a/tests/rust-python-harness/shared/parity/fixtures/recording.py b/tests/rust-python-harness/shared/parity/fixtures/recording.py new file mode 100644 index 00000000000..6c23e36f20c --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/recording.py @@ -0,0 +1,267 @@ +from __future__ import annotations + +import queue +import threading +from collections.abc import Callable, Generator, Iterable +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final, TypeVar, cast +from urllib.parse import urlsplit, urlunsplit + +import httpx +from vcr.filters import remove_query_parameters +from vcr.request import Request + +from ..http import ( + dropped_request_headers, + dropped_response_headers, + is_streaming_response, +) +from ..recorded_http import ( + HttpHeader, + RecordedHttpResponse, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) + +_PARITY_PROVIDER_HOST: Final = "parity-provider.invalid" +_SECRET_HEADERS: Final = frozenset( + { + "authorization", + "proxy-authorization", + "cookie", + "x-api-key", + "api-key", + "anthropic-api-key", + "openai-api-key", + "azure-api-key", + "x-goog-api-key", + "ocp-apim-subscription-key", + "x-amz-security-token", + } +) + +InputT = TypeVar("InputT") + + +@dataclass(frozen=True, slots=True) +class UpstreamEndpoint: + base_url: str + + +@dataclass(frozen=True, slots=True) +class RecordedInteraction: + request: Request + response: RecordedResponse + + +def _end_to_end_headers(headers: httpx.Headers) -> tuple[HttpHeader, ...]: + decoded: Final = tuple((name.decode("ascii"), value.decode("latin-1")) for name, value in headers.raw) + excluded: Final = dropped_response_headers(decoded) + return tuple( + HttpHeader(name=name, value=_normalized_response_header(name, value)) + for name, value in decoded + if name.lower() not in excluded + ) + + +def _normalized_response_header(name: str, value: str) -> str: + if name.lower() not in {"location", "operation-location"}: + return value + parsed: Final = urlsplit(value) + if not parsed.netloc: + return value + return urlunsplit(("http", _PARITY_PROVIDER_HOST, parsed.path, parsed.query, parsed.fragment)) + + +def local_response_header(name: str, value: str, provider_url: str) -> str: + if name.lower() not in {"location", "operation-location"}: + return value + parsed: Final = urlsplit(value) + if parsed.hostname != _PARITY_PROVIDER_HOST: + return value + return f"{provider_url}{parsed.path}{'?' + parsed.query if parsed.query else ''}" + + +class _RecordingProvider(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, spec: UpstreamEndpoint) -> None: + super().__init__(("127.0.0.1", 0), _RecordingHandler) + self.spec: Final = spec + self.interactions: queue.Queue[RecordedInteraction] = queue.Queue() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def take_interactions(self) -> tuple[RecordedInteraction, ...]: + try: + first: Final = self.interactions.get(timeout=5) + except queue.Empty as error: + raise RuntimeError("successful SDK call did not produce a recorded response") from error + remaining: Final = tuple(self.interactions.get_nowait() for _ in range(self.interactions.qsize())) + return (first, *remaining) + + +class _RecordingHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + self._forward() + + def do_GET(self) -> None: + self._forward() + + def do_PUT(self) -> None: + self._forward() + + def do_PATCH(self) -> None: + self._forward() + + def do_DELETE(self) -> None: + self._forward() + + def _forward(self) -> None: + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + length: Final = int(self.headers.get("content-length") or "0") + request_body: Final = self.rfile.read(length) if length else b"" + raw_headers: Final = tuple(self.headers.raw_items()) + excluded: Final = dropped_request_headers(raw_headers) + forwarded_headers: Final = tuple((name, value) for name, value in raw_headers if name.lower() not in excluded) + upstream_url: Final = f"{provider.spec.base_url.rstrip('/')}{self.path}" + + try: + with httpx.stream( + self.command, + upstream_url, + headers=forwarded_headers, + content=request_body, + timeout=120, + ) as upstream: + headers: Final = _end_to_end_headers(upstream.headers) + recorded_response: Final = self._record_upstream_response(upstream, headers) + except httpx.HTTPError as error: + self._send_response(502, (), str(error).encode("utf-8")) + return + + recorded_request: Final = remove_query_parameters( + Request( + self.command, + f"http://{_PARITY_PROVIDER_HOST}{self.path}", + request_body, + {name: value for name, value in forwarded_headers if name.lower() not in _SECRET_HEADERS}, + ), + ("api_key", "api-key", "key", "access_token", "subscription-key"), + ) + provider.interactions.put(RecordedInteraction(recorded_request, recorded_response)) + if isinstance(recorded_response, RecordedHttpResponse): + self._send_response( + recorded_response.status_code, recorded_response.headers, recorded_response.body_bytes() + ) + + def _record_upstream_response( + self, + upstream: httpx.Response, + headers: tuple[HttpHeader, ...], + ) -> RecordedResponse: + content_type: Final = cast(str, upstream.headers.get("content-type", "")) + if is_streaming_response(content_type): + return self._record_stream(upstream, headers) + response_body: Final = b"".join(upstream.iter_bytes()) + return RecordedHttpResponse.from_bytes( + status_code=upstream.status_code, + headers=headers, + body=response_body, + ) + + def _record_stream( + self, + upstream: httpx.Response, + headers: tuple[HttpHeader, ...], + ) -> RecordedHttpStreamResponse: + self.send_response_only(upstream.status_code) + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + for header in headers: + self.send_header(header.name, local_response_header(header.name, header.value, provider.url)) + self.send_header("transfer-encoding", "chunked") + self.end_headers() + chunks: Final = tuple(self._relay_chunks(upstream.iter_bytes())) + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + return RecordedHttpStreamResponse( + kind="http_stream", + status_code=upstream.status_code, + headers=headers, + chunks=chunks, + ) + + def _relay_chunks(self, chunks: Iterable[bytes]) -> Generator[RecordedStreamChunk, None, None]: + for chunk in chunks: + self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii")) + self.wfile.write(chunk) + self.wfile.write(b"\r\n") + self.wfile.flush() + yield RecordedStreamChunk.from_bytes(chunk) + + def _send_response(self, status_code: int, headers: tuple[HttpHeader, ...], body: bytes) -> None: + self.send_response_only(status_code) + provider: Final = self.server + assert isinstance(provider, _RecordingProvider) + for header in headers: + self.send_header(header.name, local_response_header(header.name, header.value, provider.url)) + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def _recording_provider(spec: UpstreamEndpoint) -> Generator[_RecordingProvider]: + server: Final = _RecordingProvider(spec) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _invoke_and_take_interactions( + recorder: _RecordingProvider, + case_input: InputT, + sdk_call: Callable[[str, InputT], object], +) -> tuple[RecordedInteraction, ...]: + try: + sdk_call(recorder.url, case_input) + except Exception as invocation_error: + try: + return recorder.take_interactions() + except RuntimeError: + raise invocation_error + return recorder.take_interactions() + + +def record_upstream_interactions( + spec: UpstreamEndpoint, + case_input: InputT, + sdk_call: Callable[[str, InputT], object], +) -> tuple[RecordedInteraction, ...]: + with _recording_provider(spec) as recorder: + return _invoke_and_take_interactions(recorder, case_input, sdk_call) + + +def record_upstream_responses( + spec: UpstreamEndpoint, + case_input: InputT, + sdk_call: Callable[[str, InputT], object], +) -> tuple[RecordedResponse, ...]: + return tuple(item.response for item in record_upstream_interactions(spec, case_input, sdk_call)) diff --git a/tests/rust-python-harness/shared/parity/fixtures/store.py b/tests/rust-python-harness/shared/parity/fixtures/store.py new file mode 100644 index 00000000000..7a10c5c6c5d --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/store.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import hashlib +import json +import tempfile +from collections.abc import Mapping +from datetime import datetime, timezone +from pathlib import Path +from typing import Final, Literal, Protocol, TypeVar, cast + +from pydantic import AwareDatetime, BaseModel, ConfigDict, TypeAdapter, ValidationError + +from .cassette import deserialize_cassette, serialize_cassette +from .recording import RecordedInteraction + +FIXTURE_SCHEMA_VERSION: Final = 1 +JSON_OBJECT: Final = TypeAdapter(dict[str, object]) + + +class FixtureInput(Protocol): + def canonical_input(self) -> dict[str, object]: ... + + +CaseT = TypeVar("CaseT", bound=BaseModel) + + +class FixtureEnvelope(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + schema_version: int + recorded_at: AwareDatetime + case: dict[str, object] + + +def canonical_json(value: Mapping[str, object]) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) + + +def fixture_cache_key(case_input: FixtureInput) -> dict[str, object]: + return case_input.canonical_input() + + +def fixture_path(directory: Path, case_input: FixtureInput) -> Path: + input_json: Final = canonical_json(fixture_cache_key(case_input)) + digest: Final = hashlib.sha256(input_json.encode("utf-8")).hexdigest() + return directory / f"{digest}.yaml" + + +def load_fixture(directory: Path, case_input: FixtureInput, case_type: type[CaseT]) -> CaseT | None: + path: Final = fixture_path(directory, case_input) + if path.is_file(): + return read_fixture(path, case_type) + legacy_path: Final = path.with_suffix(".json") + if not legacy_path.is_file(): + return None + return read_fixture(legacy_path, case_type) + + +def save_fixture( + directory: Path, + case_input: FixtureInput, + case: BaseModel, + interactions: tuple[RecordedInteraction, ...], + *, + recorded_at: datetime | None = None, + request_source: Literal["recorded", "python_replay"] = "recorded", +) -> Path: + directory.mkdir(parents=True, exist_ok=True) + path: Final = fixture_path(directory, case_input) + serialized: Final = serialize_cassette( + cast(dict[str, object], case.model_dump(mode="json", exclude_unset=True)), + interactions, + recorded_at or datetime.now(timezone.utc), + request_source, + ) + with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=directory, delete=False) as temporary: + temporary_path: Final = Path(temporary.name) + try: + temporary.write(serialized) + temporary.close() + temporary_path.replace(path) + finally: + temporary_path.unlink(missing_ok=True) + return path + + +def read_fixture(path: Path, case_type: type[CaseT]) -> CaseT: + contents: Final = path.read_text(encoding="utf-8") + if path.suffix == ".json": + return _load_fixture(JSON_OBJECT.validate_json(contents), path, case_type) + try: + cassette: Final = deserialize_cassette(contents) + return case_type.model_validate(cassette.case_data()) + except ValueError as error: + raise ValueError(f"invalid parity cassette {path}") from error + + +def _load_fixture(raw_fixture: dict[str, object], path: Path, case_type: type[CaseT]) -> CaseT: + schema_version: Final = raw_fixture.get("schema_version") + if schema_version != FIXTURE_SCHEMA_VERSION: + raise ValueError( + f"fixture {path} has schema_version {schema_version!r}, expected {FIXTURE_SCHEMA_VERSION}; " + "delete it and regenerate the fixture bundle" + ) + try: + envelope: Final = FixtureEnvelope.model_validate(raw_fixture) + return case_type.model_validate(envelope.case) + except ValidationError as error: + raise ValueError(f"invalid parity fixture {path} ({len(error.errors())} validation errors)") from error + + +def recorded_fixtures(directory: Path, case_type: type[CaseT]) -> tuple[CaseT, ...]: + if not directory.is_dir(): + return () + paths: Final = tuple(sorted((*directory.rglob("*.yaml"), *directory.rglob("*.json")))) + return tuple(read_fixture(path, case_type) for path in paths) + + +def fixture_directory(configured: Path | None, env_value: str | None, default: Path) -> Path: + return (configured or Path(env_value or default)).expanduser() + + +def fixture_id(case_input: FixtureInput, prefix: str) -> str: + input_json: Final = canonical_json(case_input.canonical_input()) + digest: Final = hashlib.sha256(input_json.encode("utf-8")).hexdigest()[:8] + return f"{prefix}-{digest}" diff --git a/tests/rust-python-harness/shared/parity/fixtures/test_cassette.py b/tests/rust-python-harness/shared/parity/fixtures/test_cassette.py new file mode 100644 index 00000000000..e668223782b --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/test_cassette.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from pathlib import Path +from typing import Final + +import httpx +import pytest +from vcr import VCR +from vcr.request import Request + +from ..fixture_models import ParityCase, SdkInputBase +from .cassette import deserialize_cassette +from .recording import RecordedInteraction +from .store import load_fixture, save_fixture +from ..recorded_http import ( + HttpHeader, + RecordedHttpResponse, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) +from ..replay import replay_server + +_URI: Final = "http://parity-provider.invalid/operation?api-version=1" + + +class _Input(SdkInputBase): + model: str = "fixture-model" + + +@pytest.mark.parametrize("body", (b'{"text":"caf\xc3\xa9"}', b"\x00\xff\x80", b"")) +def test_cassette_replays_repeated_requests_with_vcr_and_preserves_bytes(tmp_path: Path, body: bytes) -> None: + sdk_input: Final = _Input() + responses: Final = tuple( + RecordedHttpResponse.from_bytes( + status, + (HttpHeader(name="content-type", value="application/octet-stream"),), + body, + ) + for status in (200, 429) + ) + case: Final = ParityCase[_Input](litellm_input=sdk_input, provider_responses=responses) + interactions: Final = tuple( + RecordedInteraction(Request("POST", _URI, b"\xffrequest", {}), response) for response in responses + ) + timestamp: Final = datetime(2020, 1, 1, tzinfo=timezone.utc) + path: Final = save_fixture(tmp_path, sdk_input, case, interactions, recorded_at=timestamp) + + assert load_fixture(tmp_path, sdk_input, ParityCase[_Input]) == case + assert deserialize_cassette(path.read_text()).recorded_at == timestamp + with VCR().use_cassette(str(path), record_mode="none", match_on=("method", "uri", "body")) as cassette: + for status in (200, 429): + replayed: Final = httpx.post(_URI, content=b"\xffrequest") + assert replayed.status_code == status + assert replayed.content == body + assert cassette.all_played + + +def test_stream_cassette_preserves_chunk_boundaries_through_local_replay(tmp_path: Path) -> None: + sdk_input: Final = _Input() + chunks: Final = (b"data: caf\xc3", b"\xa9\n\n", b"data: [DONE]\n\n") + response: Final = RecordedHttpStreamResponse( + kind="http_stream", + status_code=200, + headers=(HttpHeader(name="content-type", value="text/event-stream"),), + chunks=tuple(RecordedStreamChunk.from_bytes(chunk) for chunk in chunks), + ) + case: Final = ParityCase[_Input](litellm_input=sdk_input, provider_responses=(response,)) + path: Final = save_fixture( + tmp_path, sdk_input, case, (RecordedInteraction(Request("POST", _URI, b"{}", {}), response),) + ) + loaded: Final = load_fixture(tmp_path, sdk_input, ParityCase[_Input]) + assert loaded == case + with replay_server() as server: + server.enqueue_response(loaded.provider_responses[0]) + with httpx.stream("POST", f"{server.url}/operation", content=b"{}") as replayed: + assert tuple(replayed.iter_raw()) == chunks + server.take_requests(1) + path.write_text(path.read_text().replace("- 10\n", "- 999\n")) + with pytest.raises(ValueError, match="invalid parity cassette"): + load_fixture(tmp_path, sdk_input, ParityCase[_Input]) + + +def test_cassette_preserves_duplicate_response_headers(tmp_path: Path) -> None: + sdk_input: Final = _Input() + response: Final[RecordedResponse] = RecordedHttpResponse.from_bytes( + 200, + (HttpHeader(name="x-test", value="first"), HttpHeader(name="x-test", value="second")), + b"{}", + ) + case: Final = ParityCase[_Input](litellm_input=sdk_input, provider_responses=(response,)) + save_fixture(tmp_path, sdk_input, case, (RecordedInteraction(Request("POST", _URI, b"", {}), response),)) + + assert load_fixture(tmp_path, sdk_input, ParityCase[_Input]) == case diff --git a/tests/rust-python-harness/shared/parity/fixtures/test_inputs.py b/tests/rust-python-harness/shared/parity/fixtures/test_inputs.py new file mode 100644 index 00000000000..d76bcac7736 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/test_inputs.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import Final + +from hypothesis import strategies as st +from pydantic import BaseModel, ConfigDict + +from .inputs import generate_case_inputs + + +class _Input(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + identifier: str + + +def test_generate_case_inputs_is_deterministic() -> None: + strategy: Final = st.builds(_Input, identifier=st.integers().map(str)) + + assert generate_case_inputs(strategy, examples=4) == generate_case_inputs(strategy, examples=4) diff --git a/tests/rust-python-harness/shared/parity/fixtures/test_media.py b/tests/rust-python-harness/shared/parity/fixtures/test_media.py new file mode 100644 index 00000000000..85518971ed4 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/test_media.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import base64 +from io import BytesIO +from typing import Final, cast + +from PIL import Image + +from .media import dummy_image_url, structured_image_bytes, structured_image_data_uri + + +def test_dummy_image_url_encodes_text_and_dimensions() -> None: + assert dummy_image_url("invoice 123", 24, width=320, height=80) == ( + "https://dummyjson.com/image/320x80/ffffff/000000?text=invoice%20123&fontSize=24" + ) + + +def test_structured_image_is_local_content_bearing_png() -> None: + png: Final = structured_image_bytes() + encoded: Final = structured_image_data_uri().partition(",")[2] + image: Final = Image.open(BytesIO(png)) + colors: Final = cast(list[tuple[int, tuple[int, int, int]]], image.getcolors(maxcolors=2)) + + assert png.startswith(b"\x89PNG\r\n\x1a\n") + assert base64.b64decode(encoded, validate=True) == png + assert image.size == (320, 80) + assert {color for _, color in colors} == {(0, 0, 0), (255, 255, 255)} diff --git a/tests/rust-python-harness/shared/parity/fixtures/test_pipeline.py b/tests/rust-python-harness/shared/parity/fixtures/test_pipeline.py new file mode 100644 index 00000000000..d865f34eeae --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/test_pipeline.py @@ -0,0 +1,191 @@ +from __future__ import annotations + +import logging +import threading +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final, Literal + +import httpx +import pytest +from hypothesis import strategies as st +from pydantic import BaseModel, ConfigDict + +from .pipeline import ( + RecordingInvocation, + RecordingTarget, + build_recording_jobs, + record_fixtures, +) +from .recording import UpstreamEndpoint +from .store import fixture_path +from ..recorded_http import RecordedResponse + + +class _FixtureInput(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + identifier: str + + def canonical_input(self) -> dict[str, object]: + return {"identifier": self.identifier} + + +class _ParityCase(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + litellm_input: _FixtureInput + provider_responses: tuple[RecordedResponse, ...] + + +class _Upstream(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _UpstreamHandler) + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + +class _UpstreamHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + length: Final = int(self.headers.get("content-length") or "0") + self.rfile.read(length) + body: Final = b"{}" + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def _upstream() -> Generator[_Upstream]: + server: Final = _Upstream() + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +@dataclass(frozen=True, slots=True) +class _OrderedInvocation: + order: Literal["slow", "fast"] + slow_started: threading.Event + fast_finished: threading.Event + + def execute(self, provider_url: str, case_input: _FixtureInput) -> None: + if self.order == "slow": + self.slow_started.set() + if not self.fast_finished.wait(timeout=2): + raise TimeoutError("fast recording did not finish") + else: + if not self.slow_started.wait(timeout=2): + raise TimeoutError("slow recording did not start") + response: Final = httpx.post(f"{provider_url}/record", json={"id": case_input.identifier}, timeout=5) + response.raise_for_status() + if self.order == "fast": + self.fast_finished.set() + + +@dataclass(frozen=True, slots=True) +class _Invocation: + def execute(self, provider_url: str, case_input: _FixtureInput) -> None: + response: Final = httpx.post(f"{provider_url}/record", json={"id": case_input.identifier}, timeout=5) + response.raise_for_status() + + +def _target( + name: str, + upstream_url: str, + case_input: _FixtureInput, + invocation: RecordingInvocation[_FixtureInput], +) -> RecordingTarget[_FixtureInput]: + return RecordingTarget( + name=name, + upstream=UpstreamEndpoint(base_url=upstream_url), + strategy=st.just(case_input), + invocation=invocation, + required_inputs=(case_input,), + ) + + +def test_build_jobs_keeps_required_inputs_before_generated_inputs_and_deduplicates(tmp_path: Path) -> None: + required: Final = _FixtureInput(identifier="required") + generated: Final = _FixtureInput(identifier="generated") + target: Final = RecordingTarget( + name="ordered", + upstream=UpstreamEndpoint(base_url="https://provider.invalid"), + strategy=st.just(generated), + invocation=_Invocation(), + required_inputs=(required, required), + ) + + jobs: Final = build_recording_jobs((target,), tmp_path, examples=1) + + assert tuple(job.case_input.identifier for job in jobs) == ("required", "generated") + + +def test_progress_follows_completion_order(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + slow_started: Final = threading.Event() + fast_finished: Final = threading.Event() + with _upstream() as upstream: + targets: Final = ( + _target( + "slow", + upstream.url, + _FixtureInput(identifier="slow"), + _OrderedInvocation("slow", slow_started, fast_finished), + ), + _target( + "fast", + upstream.url, + _FixtureInput(identifier="fast"), + _OrderedInvocation("fast", slow_started, fast_finished), + ), + ) + with caplog.at_level(logging.INFO, logger="tests.rust-python-harness.shared.parity.fixtures.pipeline"): + summary: Final = record_fixtures(targets, tmp_path, 1, 2, _ParityCase) + + progress: Final = tuple(record.message for record in caplog.records if record.message.startswith("[")) + assert len(summary.recorded) == 2 + assert summary.exit_code == 0 + assert "recorded fast" in progress[0] + assert "recorded slow" in progress[1] + assert caplog.records[0].message == "Recording 2 fixtures across 2 targets with concurrency 2" + assert caplog.records[-1].message == "Finished 2 fixtures: 2 recorded, 0 cached, 0 failed" + + +def test_failure_does_not_stop_independent_recordings(tmp_path: Path) -> None: + stale_input: Final = _FixtureInput(identifier="stale") + stale_directory: Final = tmp_path / "stale" + stale_directory.mkdir() + fixture_path(stale_directory, stale_input).with_suffix(".json").write_text( + '{"schema_version": 0}\n', encoding="utf-8" + ) + with _upstream() as upstream: + targets: Final = ( + _target("stale", upstream.url, stale_input, _Invocation()), + _target("valid", upstream.url, _FixtureInput(identifier="valid"), _Invocation()), + ) + summary: Final = record_fixtures(targets, tmp_path, 1, 2, _ParityCase) + + assert len(summary.recorded) == 1 + assert summary.recorded[0].target_name == "valid" + assert len(summary.failed) == 1 + assert summary.failed[0].target_name == "stale" + assert summary.exit_code == 1 diff --git a/tests/rust-python-harness/shared/parity/fixtures/test_recording.py b/tests/rust-python-harness/shared/parity/fixtures/test_recording.py new file mode 100644 index 00000000000..e3da59ac4d8 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/fixtures/test_recording.py @@ -0,0 +1,574 @@ +from __future__ import annotations + +import asyncio +import queue +import threading +from collections.abc import AsyncIterator, Callable, Generator, Iterator +from contextlib import contextmanager +from dataclasses import dataclass +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Final, Literal + +import httpx +import pytest +from hypothesis import strategies as st +from openai._streaming import SSEDecoder +from pydantic import BaseModel, ConfigDict + +from ..compare import assert_request_parity +from .pipeline import RecordingTarget, record_fixtures +from .recording import ( + UpstreamEndpoint, + record_upstream_interactions, + record_upstream_responses, +) +from .store import ( + FIXTURE_SCHEMA_VERSION, + fixture_path, + load_fixture, + recorded_fixtures, +) +from ..inprocess import InProcessExecution, run_in_process, run_in_process_async +from ..recorded_http import ( + HttpHeader, + RecordedHttpStreamResponse, + RecordedResponse, + RecordedStreamChunk, +) +from ..replay import ReplayServer, replay_server +from ..stream import ( + StreamCompleted, + StreamFailed, + StreamOutcome, + assert_stream_parity, + consume_async_stream, + consume_sync_stream, +) + +_SSE_CHUNKS: Final = ( + b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n', + b'data: {"choices":[{"delta":{"content":" world"}}]}\n\n', + b"data: [DONE]\n\n", +) + + +class _StreamEvent(BaseModel): + kind: Literal["delta", "done", "error"] + value: str + + +class _StreamApplicationError(Exception): + status_code: Final = 400 + code: Final = "invalid_input" + type: Final = "validation_error" + param: Final = "input" + model: Final = "fixture-model" + llm_provider: Final = "fixture-provider" + + +def _stream_event(data: str) -> _StreamEvent: + event: Final = _StreamEvent.model_validate_json(data) + if event.kind == "error": + raise _StreamApplicationError(event.value) + return event + + +def _event_chunks(failed: bool) -> tuple[bytes, ...]: + terminal: Final = ( + b'event: error\r\ndata: {"kind":"error","value":"invalid input"}\r\n\r\n' + if failed + else b'event: done\r\ndata: {"kind":"done","value":""}\r\n\r\n' + ) + return ( + b'event: delta\r\ndata: {"kind":"delta",\r\ndata: "value":"caf\xc3', + b'\xa9"}\r\n', + b'\r\nevent: delta\r\ndata: {"kind":"delta","value":"second"}\r\n\r\n' + terminal, + ) + + +def _sync_events(api_base: str, case_input: _FixtureInput) -> Iterator[_StreamEvent]: + with httpx.stream("POST", f"{api_base}/stream", json={"id": case_input.identifier}, timeout=5) as response: + response.raise_for_status() + for event in SSEDecoder().iter_bytes(response.iter_bytes()): + yield _stream_event(event.data) + + +async def _async_events(api_base: str, case_input: _FixtureInput) -> AsyncIterator[_StreamEvent]: + async with httpx.AsyncClient(timeout=5) as client: + async with client.stream("POST", f"{api_base}/stream", json={"id": case_input.identifier}) as response: + response.raise_for_status() + async for event in SSEDecoder().aiter_bytes(response.aiter_bytes()): + yield _stream_event(event.data) + + +async def _consume_async_events(api_base: str, case_input: _FixtureInput) -> StreamOutcome: + async def create() -> AsyncIterator[_StreamEvent]: + return _async_events(api_base, case_input) + + return await consume_async_stream(create) + + +async def _replay_events( + mode: Literal["sync", "async"], + provider: ReplayServer, + response: RecordedHttpStreamResponse, + case_input: _FixtureInput, +) -> InProcessExecution[StreamOutcome]: + if mode == "sync": + return run_in_process( + provider, (response,), lambda url: consume_sync_stream(lambda: _sync_events(url, case_input)) + ) + return await run_in_process_async(provider, (response,), lambda url: _consume_async_events(url, case_input)) + + +class _FixtureInput(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + identifier: str + + def canonical_input(self) -> dict[str, object]: + return {"identifier": self.identifier} + + +class _ParityCase(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + litellm_input: _FixtureInput + provider_responses: tuple[RecordedResponse, ...] + + +@dataclass(frozen=True, slots=True) +class _Invocation: + sdk_call: Callable[[str, _FixtureInput], object] + + def execute(self, provider_url: str, case_input: _FixtureInput) -> None: + self.sdk_call(provider_url, case_input) + + +class _ControlledUpstream(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, stream_chunks: tuple[bytes, ...]) -> None: + super().__init__(("127.0.0.1", 0), _ControlledUpstreamHandler) + self.stream_chunks: Final = stream_chunks + self.lock: Final = threading.Lock() + self.two_requests_started: Final = threading.Event() + self.active_requests: int = 0 + self.max_active_requests: int = 0 + self.request_count: int = 0 + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def start_request(self) -> None: + with self.lock: + self.active_requests += 1 + self.request_count += 1 + self.max_active_requests = max(self.max_active_requests, self.active_requests) + if self.active_requests == 2: + self.two_requests_started.set() + self.two_requests_started.wait(timeout=2) + + def end_tracked_request(self) -> None: + with self.lock: + self.active_requests -= 1 + + +class _ControlledUpstreamHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + upstream: Final = self.server + assert isinstance(upstream, _ControlledUpstream) + length: Final = int(self.headers.get("content-length") or "0") + self.rfile.read(length) + if self.path == "/credentials?api_key=query-secret&api-version=1": + authorized: Final = self.headers.get("authorization") == "Bearer header-secret" + self._send_json(200 if authorized else 401, b"{}") + return + if self.path == "/upload": + self._send_json(200, b'{"file_id":"fixture://document.pdf"}') + return + if self.path == "/parse": + self._send_json(200, b'{"result":{"chunks":[]}}') + return + if self.path == "/analyze": + self.send_response(202) + self.send_header("operation-location", f"{upstream.url}/results/1") + self.send_header("content-length", "0") + self.end_headers() + return + if self.path in {"/v1/chat/completions", "/stream"}: + with upstream.lock: + upstream.request_count += 1 + self.send_response(200) + self.send_header("content-type", "text/event-stream") + self.send_header("transfer-encoding", "chunked") + self.end_headers() + for chunk in upstream.stream_chunks: + self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii")) + self.wfile.write(chunk) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + return + if self.path == "/error": + self._send_json(429, b'{"error":{"message":"rate limited"}}') + return + upstream.start_request() + try: + body: Final = b"{}" + self.send_response(200) + self.send_header("content-type", "application/json") + self.send_header("set-cookie", "session=must-not-be-recorded") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + finally: + upstream.end_tracked_request() + + def do_GET(self) -> None: + if self.path == "/results/1": + self._send_json(200, b'{"status":"succeeded","analyzeResult":{"pages":[]}}') + return + self.send_error(404) + + def do_PUT(self) -> None: + self.do_POST() + + def do_PATCH(self) -> None: + self.do_POST() + + def do_DELETE(self) -> None: + self.do_POST() + + def _send_json(self, status: int, body: bytes) -> None: + self.send_response(status) + self.send_header("content-type", "application/json") + self.send_header("content-length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def _controlled_upstream(stream_chunks: tuple[bytes, ...] = _SSE_CHUNKS) -> Generator[_ControlledUpstream]: + server: Final = _ControlledUpstream(stream_chunks) + thread: Final = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) + + +def _case(identifier: str) -> _FixtureInput: + return _FixtureInput(identifier=identifier) + + +def _sdk_call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.post(f"{api_base}/v1/operation", content=b"{}", timeout=5) + + +def _stream_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.post(f"{api_base}/v1/chat/completions", content=b"{}", timeout=5) + + +def _error_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + response: Final = httpx.post(f"{api_base}/error", content=b"{}", timeout=5) + response.raise_for_status() + return response + + +def _method_sdk_call(method: str) -> Callable[[str, _FixtureInput], object]: + def call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.request(method, f"{api_base}/method", json={"id": case_input.identifier}, timeout=5) + + return call + + +def _multi_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + upload: Final = httpx.post(f"{api_base}/upload", json={"document": case_input.identifier}, timeout=5) + upload.raise_for_status() + parsed: Final = httpx.post(f"{api_base}/parse", json={"input": upload.json()["file_id"]}, timeout=5) + parsed.raise_for_status() + return parsed + + +def _polling_sdk_call(api_base: str, case_input: _FixtureInput) -> object: + started: Final = httpx.post(f"{api_base}/analyze", json={"document": case_input.identifier}, timeout=5) + operation_location: Final = started.headers["operation-location"] + completed: Final = httpx.get(operation_location, timeout=5) + completed.raise_for_status() + return completed + + +def test_recording_deduplicates_per_target_and_caps_global_concurrency(tmp_path: Path) -> None: + shared_input: Final = _case("shared") + with _controlled_upstream() as upstream: + spec: Final = UpstreamEndpoint(base_url=upstream.url) + targets: Final = ( + RecordingTarget( + name="first", + upstream=spec, + strategy=st.just(shared_input), + invocation=_Invocation(_sdk_call), + required_inputs=(shared_input, shared_input), + ), + RecordingTarget( + name="second", + upstream=spec, + strategy=st.just(shared_input), + invocation=_Invocation(_sdk_call), + required_inputs=(shared_input,), + ), + ) + summary: Final = record_fixtures(targets, tmp_path, examples=1, concurrency=2, case_type=_ParityCase) + + assert len(summary.recorded) == 2 + assert {result.target_name for result in summary.recorded} == {"first", "second"} + assert summary.cached == () + assert summary.failed == () + assert upstream.request_count == 2 + assert upstream.max_active_requests == 2 + assert len(recorded_fixtures(tmp_path, _ParityCase)) == 2 + for path in tmp_path.rglob("*.yaml"): + contents = path.read_text(encoding="utf-8") + assert f"schema_version: {FIXTURE_SCHEMA_VERSION}" in contents + assert "recorded_at:" in contents + + +def test_pipeline_rejects_stale_fixture_before_provider_call(tmp_path: Path) -> None: + case_input: Final = _case("stale") + directory: Final = tmp_path / "stale-target" + directory.mkdir() + path: Final = fixture_path(directory, case_input).with_suffix(".json") + path.write_text('{"schema_version": 0}\n', encoding="utf-8") + target: Final = RecordingTarget( + name="stale-target", + upstream=UpstreamEndpoint(base_url="http://127.0.0.1:1"), + strategy=st.just(case_input), + invocation=_Invocation(_sdk_call), + ) + + summary: Final = record_fixtures( + (target,), + tmp_path, + examples=1, + concurrency=1, + case_type=_ParityCase, + ) + + assert summary.recorded == () + assert summary.cached == () + assert len(summary.failed) == 1 + assert str(summary.failed[0].error) == ( + f"fixture {path} has schema_version 0, expected {FIXTURE_SCHEMA_VERSION}; " + "delete it and regenerate the fixture bundle" + ) + + +def test_cached_fixture_is_reported_without_provider_call(tmp_path: Path) -> None: + case_input: Final = _case("cached") + with _controlled_upstream() as upstream: + target: Final = RecordingTarget( + name="cached-target", + upstream=UpstreamEndpoint(base_url=upstream.url), + strategy=st.just(case_input), + invocation=_Invocation(_sdk_call), + ) + first: Final = record_fixtures((target,), tmp_path, 1, 1, _ParityCase) + second: Final = record_fixtures((target,), tmp_path, 1, 1, _ParityCase) + + assert len(first.recorded) == 1 + assert len(second.cached) == 1 + assert upstream.request_count == 1 + + +def test_streaming_response_records_and_replays_chunks() -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case("stream"), + _stream_sdk_call, + ) + + response: Final = responses[0] + assert isinstance(response, RecordedHttpStreamResponse) + assert tuple(chunk.data_bytes() for chunk in response.chunks) == _SSE_CHUNKS + assert isinstance(response.model_dump(mode="json")["chunks"], list) + + with replay_server() as provider: + provider.enqueue_response(response) + with httpx.stream("POST", f"{provider.url}/v1/chat/completions", json={}) as replayed: + replayed_chunks: Final = tuple(replayed.iter_raw()) + provider.take_requests(1) + + assert replayed_chunks == _SSE_CHUNKS + + +def test_non_successful_provider_response_is_recorded() -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case("provider-error"), + _error_sdk_call, + ) + + response: Final = responses[0] + assert response.status_code == 429 + + +def test_sensitive_response_headers_are_not_recorded() -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case("headers"), + _sdk_call, + ) + + assert all(header.name.lower() != "set-cookie" for header in responses[0].headers) + + +def test_recorded_requests_strip_credentials_without_changing_the_live_request() -> None: + def sdk_call(api_base: str, case_input: _FixtureInput) -> object: + return httpx.post( + f"{api_base}/credentials?api_key=query-secret&api-version=1", + headers={ + "Authorization": "Bearer header-secret", + "Ocp-Apim-Subscription-Key": "azure-secret", + "Cookie": "session=cookie-secret", + "X-Test": case_input.identifier, + }, + content=b"\xffdocument", + ) + + with _controlled_upstream() as upstream: + interactions: Final = record_upstream_interactions( + UpstreamEndpoint(upstream.url), _case("credentials"), sdk_call + ) + + interaction: Final = interactions[0] + assert interaction.response.status_code == 200 + assert interaction.request.uri == "http://parity-provider.invalid/credentials?api-version=1" + assert interaction.request.body == b"\xffdocument" + assert interaction.request.headers["x-test"] == "credentials" + assert all( + header not in interaction.request.headers for header in ("authorization", "ocp-apim-subscription-key", "cookie") + ) + + +@pytest.mark.parametrize("method", ("PUT", "PATCH", "DELETE")) +def test_recording_and_replay_support_mutating_http_methods(method: str) -> None: + sdk_call: Final = _method_sdk_call(method) + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case(method), + sdk_call, + ) + with replay_server() as provider: + provider.enqueue_response(responses[0]) + sdk_call(provider.url, _case(method)) + requests: Final = provider.take_requests(1) + + assert requests[0].method == method + + +def test_stream_response_model_rejects_buffered_body() -> None: + with pytest.raises(ValueError, match="Extra inputs are not permitted"): + RecordedHttpStreamResponse.model_validate( + { + "kind": "http_stream", + "status_code": 200, + "headers": [HttpHeader(name="content-type", value="text/event-stream")], + "chunks": [RecordedStreamChunk.from_bytes(b"data: [DONE]\n\n")], + "body_b64": "", + } + ) + + +@pytest.mark.parametrize("sdk_call", (_multi_sdk_call, _polling_sdk_call)) +def test_multiple_provider_calls_record_and_replay_in_order( + sdk_call: Callable[[str, _FixtureInput], object], +) -> None: + with _controlled_upstream() as upstream: + responses: Final = record_upstream_responses( + UpstreamEndpoint(base_url=upstream.url), + _case(sdk_call.__name__), + sdk_call, + ) + + assert len(responses) == 2 + with replay_server() as provider: + for response in responses: + provider.enqueue_response(response) + sdk_call(provider.url, _case(sdk_call.__name__)) + requests: Final = provider.take_requests(2) + + assert len(requests) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize("failed", (False, True), ids=("completed", "application-error")) +async def test_typed_stream_recording_cassette_replay_parity( + tmp_path: Path, mode: Literal["sync", "async"], failed: bool +) -> None: + case_input: Final = _case("typed-stream") + outcomes: Final[queue.SimpleQueue[StreamOutcome]] = queue.SimpleQueue() + + def record(api_base: str, sdk_input: _FixtureInput) -> None: + outcome: Final = ( + consume_sync_stream(lambda: _sync_events(api_base, sdk_input)) + if mode == "sync" + else asyncio.run(_consume_async_events(api_base, sdk_input)) + ) + outcomes.put(outcome) + + with _controlled_upstream(_event_chunks(failed)) as upstream: + target: Final = RecordingTarget( + name="stream", + upstream=UpstreamEndpoint(upstream.url), + strategy=st.just(case_input), + invocation=_Invocation(record), + ) + summary: Final = record_fixtures((target,), tmp_path, 1, 1, _ParityCase) + + assert summary.failed == () + assert len(summary.recorded) == 1 + recorded: Final = outcomes.get_nowait() + loaded: Final = load_fixture(tmp_path / "stream", case_input, _ParityCase) + assert loaded is not None + response: Final = loaded.provider_responses[0] + assert isinstance(response, RecordedHttpStreamResponse) + assert response.status_code == 200 + wire_bytes: Final = b"".join(chunk.data_bytes() for chunk in response.chunks) + assert wire_bytes == b"".join(_event_chunks(failed)) + coalesced: Final = response.model_copy(update={"chunks": (RecordedStreamChunk.from_bytes(wire_bytes),)}) + + with replay_server() as provider: + first: Final = await _replay_events(mode, provider, response, case_input) + second: Final = await _replay_events(mode, provider, coalesced, case_input) + assert_request_parity(first.requests, second.requests) + assert len(first.requests) == 1 + assert first.requests[0].body == {"id": case_input.identifier} + assert_stream_parity(recorded, first.response) + assert_stream_parity(first.response, second.response) + expected: Final = (_StreamEvent(kind="delta", value="café"), _StreamEvent(kind="delta", value="second")) + assert first.response.chunks == (expected if failed else (*expected, _StreamEvent(kind="done", value=""))) + if failed: + assert isinstance(first.response.terminal, StreamFailed) + assert first.response.terminal.phase == "iteration" + assert first.response.terminal.exception_type is _StreamApplicationError + assert first.response.terminal.error.code == "invalid_input" + assert first.response.terminal.error.message == "invalid input" + else: + assert first.response.terminal == StreamCompleted() diff --git a/tests/rust-python-harness/shared/parity/http.py b/tests/rust-python-harness/shared/parity/http.py new file mode 100644 index 00000000000..c164e3c6549 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/http.py @@ -0,0 +1,54 @@ +from __future__ import annotations + +from collections.abc import Iterable +from typing import Final + +HOP_BY_HOP_HEADERS: Final[frozenset[str]] = frozenset( + { + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "trailers", + "transfer-encoding", + "upgrade", + } +) + +REQUEST_DROPPED_HEADERS: Final[frozenset[str]] = HOP_BY_HOP_HEADERS | { + "host", + "content-length", + "accept-encoding", +} + +RESPONSE_DROPPED_HEADERS: Final[frozenset[str]] = HOP_BY_HOP_HEADERS | { + "content-encoding", + "content-length", + "set-cookie", +} + + +def connection_header_names(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + return frozenset( + token.strip().lower() + for name, value in headers + if name.lower() == "connection" + for token in value.split(",") + if token.strip() + ) + + +def dropped_request_headers(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + materialized: Final = tuple(headers) + return REQUEST_DROPPED_HEADERS | connection_header_names(materialized) + + +def dropped_response_headers(headers: Iterable[tuple[str, str]]) -> frozenset[str]: + materialized: Final = tuple(headers) + return RESPONSE_DROPPED_HEADERS | connection_header_names(materialized) + + +def is_streaming_response(content_type: str) -> bool: + return "text/event-stream" in content_type.lower() diff --git a/tests/rust-python-harness/shared/parity/inprocess.py b/tests/rust-python-harness/shared/parity/inprocess.py new file mode 100644 index 00000000000..36a73db9239 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/inprocess.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass +from typing import Final, Generic, TypeVar + +from .models import CapturedRequest +from .recorded_http import RecordedResponse +from .replay import ReplayServer + +ResponseT = TypeVar("ResponseT") + + +@dataclass(frozen=True, slots=True) +class InProcessExecution(Generic[ResponseT]): + requests: tuple[CapturedRequest, ...] + response: ResponseT + + +def run_in_process( + provider: ReplayServer, + recorded_responses: tuple[RecordedResponse, ...], + call: Callable[[str], ResponseT], +) -> InProcessExecution[ResponseT]: + for recorded_response in recorded_responses: + provider.enqueue_response(recorded_response) + try: + response: Final = call(provider.url) + return InProcessExecution(requests=provider.take_requests(len(recorded_responses)), response=response) + except Exception: + provider.reset() + raise + + +async def run_in_process_async( + provider: ReplayServer, + recorded_responses: tuple[RecordedResponse, ...], + call: Callable[[str], Awaitable[ResponseT]], +) -> InProcessExecution[ResponseT]: + for recorded_response in recorded_responses: + provider.enqueue_response(recorded_response) + try: + response: Final = await call(provider.url) + return InProcessExecution(requests=provider.take_requests(len(recorded_responses)), response=response) + except Exception: + provider.reset() + raise diff --git a/tests/rust-python-harness/shared/parity/models.py b/tests/rust-python-harness/shared/parity/models.py new file mode 100644 index 00000000000..898b58d23ee --- /dev/null +++ b/tests/rust-python-harness/shared/parity/models.py @@ -0,0 +1,145 @@ +from __future__ import annotations + +import base64 +from typing import Annotated, Final, Literal, cast + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter + + +class CapturedRequest(BaseModel): + model_config = ConfigDict(frozen=True) + + method: str + path: str + headers: tuple[tuple[str, str], ...] + body: JsonValue + user_agent: str | None + + +class SDKSuccess(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["ok"] = "ok" + response: JsonValue + + +class SDKError(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["error"] = "error" + exception_type: str + message: str + status_code: int | None + code: str | None + error_type: str | None + param: str | None + model: str | None + llm_provider: str | None + + +class SDKJsonChunk(BaseModel): + model_config = ConfigDict(frozen=True) + + kind: Literal["json"] = "json" + value: JsonValue + + +class SDKBytesChunk(BaseModel): + model_config = ConfigDict(frozen=True) + + kind: Literal["bytes"] = "bytes" + data_b64: str + + def data_bytes(self) -> bytes: + return base64.b64decode(self.data_b64, validate=True) + + +SDKChunk = Annotated[SDKJsonChunk | SDKBytesChunk, Field(discriminator="kind")] + + +class SDKStreamCompleted(BaseModel): + model_config = ConfigDict(frozen=True) + + kind: Literal["completed"] = "completed" + + +class SDKStreamFailed(BaseModel): + model_config = ConfigDict(frozen=True) + + kind: Literal["failed"] = "failed" + error: SDKError + + +SDKStreamTerminal = Annotated[SDKStreamCompleted | SDKStreamFailed, Field(discriminator="kind")] + + +class SDKStreamReport(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["stream"] = "stream" + chunks: tuple[SDKChunk, ...] + terminal: SDKStreamTerminal + + +SDKReport = Annotated[SDKSuccess | SDKError | SDKStreamReport, Field(discriminator="status")] +JSON_VALUE_ADAPTER: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) + + +def sdk_chunk(value: object) -> SDKChunk: + if isinstance(value, bytes): + return SDKBytesChunk(data_b64=base64.b64encode(value).decode("ascii")) + if isinstance(value, BaseModel): + return SDKJsonChunk(value=JSON_VALUE_ADAPTER.validate_python(value.model_dump(mode="json"))) + return SDKJsonChunk(value=JSON_VALUE_ADAPTER.validate_python(value)) + + +def _string_attribute(error: Exception, name: str) -> str | None: + value: Final = cast(object | None, getattr(error, name, None)) + return None if value is None else str(value) + + +def sdk_error_report(error: Exception) -> SDKError: + message, _, _ = str(error).partition("\nTraceback (most recent call last):") + raw_status_code: Final = cast(object | None, getattr(error, "status_code", None)) + status_code: Final = raw_status_code if isinstance(raw_status_code, int) else None + return SDKError( + exception_type=f"{type(error).__module__}.{type(error).__qualname__}", + message=message.rstrip(), + status_code=status_code, + code=_string_attribute(error, "code"), + error_type=_string_attribute(error, "type"), + param=_string_attribute(error, "param"), + model=_string_attribute(error, "model"), + llm_provider=_string_attribute(error, "llm_provider"), + ) + + +class Execution(BaseModel): + model_config = ConfigDict(frozen=True) + + requests: tuple[CapturedRequest, ...] + report: SDKReport + + +class SDKCommand(BaseModel): + model_config = ConfigDict(frozen=True) + + case_file: str + route: str + + +class WorkerSuccess(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["ok"] = "ok" + report: SDKReport + + +class WorkerFailure(BaseModel): + model_config = ConfigDict(frozen=True) + + status: Literal["error"] = "error" + error: str + + +WorkerResult = Annotated[WorkerSuccess | WorkerFailure, Field(discriminator="status")] diff --git a/tests/rust-python-harness/shared/parity/recorded_http.py b/tests/rust-python-harness/shared/parity/recorded_http.py new file mode 100644 index 00000000000..940c5211753 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/recorded_http.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import base64 +from typing import Annotated, Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class _RecordedHttpModel(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + +class HttpHeader(_RecordedHttpModel): + name: str + value: str + + +class RecordedHttpResponse(_RecordedHttpModel): + kind: Literal["http"] + status_code: int + headers: tuple[HttpHeader, ...] + body_b64: str + + @classmethod + def from_bytes( + cls, + status_code: int, + headers: tuple[HttpHeader, ...], + body: bytes, + ) -> RecordedHttpResponse: + return cls( + kind="http", + status_code=status_code, + headers=headers, + body_b64=base64.b64encode(body).decode("ascii"), + ) + + def body_bytes(self) -> bytes: + return base64.b64decode(self.body_b64, validate=True) + + +class RecordedStreamChunk(_RecordedHttpModel): + data_b64: str + + @classmethod + def from_bytes(cls, data: bytes) -> RecordedStreamChunk: + return cls(data_b64=base64.b64encode(data).decode("ascii")) + + def data_bytes(self) -> bytes: + return base64.b64decode(self.data_b64, validate=True) + + +class RecordedHttpStreamResponse(_RecordedHttpModel): + kind: Literal["http_stream"] + status_code: int + headers: tuple[HttpHeader, ...] + chunks: tuple[RecordedStreamChunk, ...] + + +RecordedResponse = Annotated[ + RecordedHttpResponse | RecordedHttpStreamResponse, + Field(discriminator="kind"), +] diff --git a/tests/rust-python-harness/shared/parity/replay.py b/tests/rust-python-harness/shared/parity/replay.py new file mode 100644 index 00000000000..bde84aba3c4 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/replay.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import base64 +import queue +import threading +from collections.abc import Generator +from contextlib import contextmanager +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Final + +from pydantic import JsonValue, TypeAdapter + +from .fixtures.recording import local_response_header +from .models import CapturedRequest +from .recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse + +JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) +EXCLUDED_REQUEST_HEADERS: Final = frozenset( + { + "host", + "content-length", + "connection", + "accept-encoding", + "user-agent", + "x-litellm-parity-route", + } +) +EXCLUDED_RESPONSE_HEADERS: Final = frozenset({"content-length", "transfer-encoding", "connection"}) + + +class ReplayServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _ReplayHandler) + self.responses: queue.Queue[RecordedResponse] = queue.Queue() + self.requests: queue.Queue[CapturedRequest] = queue.Queue() + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.server_address[1]}" + + def enqueue_response(self, response: RecordedResponse) -> None: + self.responses.put(response) + + def take_requests(self, expected_count: int) -> tuple[CapturedRequest, ...]: + request_count: Final = self.requests.qsize() + if request_count != expected_count: + raise AssertionError(f"expected exactly {expected_count} provider requests, received {request_count}") + return tuple(self.requests.get_nowait() for _ in range(request_count)) + + def reset(self) -> None: + while not self.responses.empty(): + self.responses.get_nowait() + while not self.requests.empty(): + self.requests.get_nowait() + + +class _ReplayHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + + def do_POST(self) -> None: + self._replay() + + def do_GET(self) -> None: + self._replay() + + def do_PUT(self) -> None: + self._replay() + + def do_PATCH(self) -> None: + self._replay() + + def do_DELETE(self) -> None: + self._replay() + + def _replay(self) -> None: + provider: Final = self.server + assert isinstance(provider, ReplayServer) + length: Final = int(self.headers.get("content-length") or "0") + raw_body: Final = self.rfile.read(length) if length else b"" + content_type: Final = self.headers.get("content-type", "") + body: Final = ( + JSON_VALUE.validate_json(raw_body) + if raw_body and content_type.lower().startswith("application/json") + else base64.b64encode(raw_body).decode("ascii") + if raw_body + else None + ) + headers: Final = tuple( + sorted( + (name.lower(), value) + for name, value in self.headers.raw_items() + if name.lower() not in EXCLUDED_REQUEST_HEADERS + ) + ) + provider.requests.put( + CapturedRequest( + method=self.command, + path=self.path, + headers=headers, + body=body, + user_agent=self.headers.get("user-agent"), + ) + ) + try: + response: Final = provider.responses.get(timeout=5) + except queue.Empty: + self.send_error(500, "no replay response queued") + return + self.send_response_only(response.status_code) + for header in response.headers: + if header.name.lower() not in EXCLUDED_RESPONSE_HEADERS: + self.send_header(header.name, local_response_header(header.name, header.value, provider.url)) + if isinstance(response, RecordedHttpResponse): + response_body: Final = response.body_bytes() + self.send_header("content-length", str(len(response_body))) + self.end_headers() + self.wfile.write(response_body) + return + assert isinstance(response, RecordedHttpStreamResponse) + self.send_header("transfer-encoding", "chunked") + self.end_headers() + for chunk in response.chunks: + data = chunk.data_bytes() + self.wfile.write(f"{len(data):X}\r\n".encode("ascii")) + self.wfile.write(data) + self.wfile.write(b"\r\n") + self.wfile.flush() + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + + def log_message(self, format: str, *args: object) -> None: + return + + +@contextmanager +def replay_server() -> Generator[ReplayServer]: + server: Final = ReplayServer() + thread: Final = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.01}, daemon=True) + thread.start() + try: + yield server + finally: + server.shutdown() + server.server_close() + thread.join(timeout=5) diff --git a/tests/rust-python-harness/shared/parity/runner.py b/tests/rust-python-harness/shared/parity/runner.py new file mode 100644 index 00000000000..5add5177113 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/runner.py @@ -0,0 +1,198 @@ +from __future__ import annotations + +import asyncio +import os +import subprocess +import sys +from collections import deque +from collections.abc import Callable, Generator +from concurrent.futures import ThreadPoolExecutor, TimeoutError +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Final, TextIO, cast + +from pydantic import TypeAdapter, ValidationError + +from .models import ( + Execution, + SDKCommand, + WorkerFailure, + WorkerResult, + WorkerSuccess, +) +from .recorded_http import RecordedResponse +from .replay import ReplayServer, replay_server + +WORKER_RESULT_PREFIX: Final = "LITELLM_PARITY_RESULT " +WORKER_RESULT_ADAPTER: Final[TypeAdapter[WorkerResult]] = TypeAdapter(WorkerResult) + + +@dataclass(frozen=True, slots=True) +class SubprocessRunner: + entrypoint: Path + baseline_user_agent: str + route_label: str + + def command(self, provider_url: str) -> tuple[str, ...]: + return ( + sys.executable, + "-m", + ".".join( + self.entrypoint.resolve().relative_to(Path(__file__).resolve().parents[4]).with_suffix("").parts + ), + "--parity-worker", + provider_url, + ) + + +@dataclass(frozen=True, slots=True) +class ExecutionVariant: + name: str + environment: tuple[tuple[str, str], ...] + + +class SubprocessWorker: + def __init__(self, runner: SubprocessRunner, provider: ReplayServer, variant: ExecutionVariant) -> None: + project_root: Final = str(Path(__file__).resolve().parents[4]) + existing_pythonpath: Final = os.environ.get("PYTHONPATH") + env: Final = { + **os.environ, + **dict(variant.environment), + "LITELLM_USER_AGENT": runner.baseline_user_agent, + "PYTHONPATH": os.pathsep.join(path for path in (project_root, existing_pythonpath) if path), + } + self.mode: Final = variant.name + self.route_label: Final = runner.route_label + self.provider: Final = provider + self.process: Final = subprocess.Popen( + runner.command(provider.url), + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + text=True, + bufsize=1, + env=env, + ) + self.output_reader: Final = ThreadPoolExecutor(max_workers=1) + self.recent_output: Final[deque[str]] = deque(maxlen=100) + + def execute( + self, + case_file: Path, + route: str, + responses: tuple[RecordedResponse, ...], + ) -> Execution: + stdin: Final = self.process.stdin + if stdin is None or self.process.poll() is not None: + raise AssertionError(f"{self.mode} {self.route_label} worker exited before processing {case_file}") + for response in responses: + self.provider.enqueue_response(response) + command: Final = SDKCommand(case_file=str(case_file), route=route) + try: + stdin.write(f"{command.model_dump_json()}\n") + stdin.flush() + result: Final = self.output_reader.submit(self._read_result).result(timeout=60) + except TimeoutError as error: + self.provider.reset() + self.close() + raise AssertionError( + f"{self.mode} {self.route_label} worker timed out after 60s while processing {case_file}" + ) from error + except AssertionError: + self.provider.reset() + raise + except (BrokenPipeError, OSError) as error: + self.provider.reset() + raise AssertionError(self._failure_message(f"worker pipe failed while processing {case_file}")) from error + if isinstance(result, WorkerFailure): + self.provider.reset() + raise AssertionError( + f"{self.mode} {self.route_label} worker failed while processing {case_file}:\n{result.error}" + ) + assert isinstance(result, WorkerSuccess) + try: + return Execution(requests=self.provider.take_requests(len(responses)), report=result.report) + except AssertionError: + self.provider.reset() + raise + + def _read_result(self) -> WorkerResult: + process_stdout: Final = self.process.stdout + if process_stdout is None: + raise AssertionError(self._failure_message("worker stdout is unavailable")) + stdout: Final = cast(TextIO, process_stdout) + line: Final = stdout.readline() + if not line: + raise AssertionError(self._failure_message("worker exited without returning a result")) + stripped: Final = line.rstrip() + if not stripped.startswith(WORKER_RESULT_PREFIX): + self.recent_output.append(stripped) + return self._read_result() + payload: Final = stripped.removeprefix(WORKER_RESULT_PREFIX) + try: + return WORKER_RESULT_ADAPTER.validate_json(payload) + except ValidationError as error: + raise AssertionError(self._failure_message("worker returned an invalid result")) from error + + def _failure_message(self, message: str) -> str: + output: Final = "\n".join(self.recent_output) + prefix: Final = f"{self.mode} {self.route_label}" + return f"{prefix} {message}" if not output else f"{prefix} {message}\noutput:\n{output}" + + def close(self) -> None: + stdin: Final = self.process.stdin + if stdin is not None and not stdin.closed: + stdin.close() + try: + self.process.wait(timeout=10) + except subprocess.TimeoutExpired: + self.process.terminate() + self.process.wait(timeout=10) + self.output_reader.shutdown(wait=True, cancel_futures=True) + + +@contextmanager +def execution_worker( + runner: SubprocessRunner, + variant: ExecutionVariant, +) -> Generator[SubprocessWorker]: + with replay_server() as provider: + worker: Final = SubprocessWorker(runner, provider, variant) + try: + yield worker + finally: + worker.close() + + +def run_execution( + worker: SubprocessWorker, + case_file: Path, + route: str, + responses: tuple[RecordedResponse, ...], +) -> Execution: + return worker.execute(case_file, route, responses) + + +@contextmanager +def execution_worker_pair( + runner: SubprocessRunner, + baseline: ExecutionVariant, + candidate: ExecutionVariant, +) -> Generator[tuple[SubprocessWorker, SubprocessWorker]]: + with execution_worker(runner, baseline) as baseline_worker: + with execution_worker(runner, candidate) as candidate_worker: + yield baseline_worker, candidate_worker + + +def parity_worker_main( + execute_command: Callable[[str, str, asyncio.AbstractEventLoop], WorkerResult], + mock_url: str, +) -> None: + event_loop: Final = asyncio.new_event_loop() + try: + for line in sys.stdin: + sys.stdout.write(f"{WORKER_RESULT_PREFIX}{execute_command(line, mock_url, event_loop).model_dump_json()}\n") + sys.stdout.flush() + finally: + event_loop.close() diff --git a/tests/rust-python-harness/shared/parity/stream.py b/tests/rust-python-harness/shared/parity/stream.py new file mode 100644 index 00000000000..72e00d3b2bd --- /dev/null +++ b/tests/rust-python-harness/shared/parity/stream.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +from collections.abc import AsyncIterable, Awaitable, Callable, Iterable +from dataclasses import dataclass +from typing import Final, Literal, TypeAlias + +from .compare import assert_value_parity +from .models import ( + SDKError, + SDKReport, + SDKStreamCompleted, + SDKStreamFailed, + SDKStreamReport, + sdk_chunk, + sdk_error_report, +) + + +@dataclass(frozen=True, slots=True) +class StreamCompleted: + kind: Literal["completed"] = "completed" + + +@dataclass(frozen=True, slots=True) +class StreamFailed: + phase: Literal["creation", "iteration"] + exception_type: type[BaseException] + error: SDKError + kind: Literal["failed"] = "failed" + + +StreamTerminal: TypeAlias = StreamCompleted | StreamFailed + + +@dataclass(frozen=True, slots=True) +class StreamOutcome: + wrapper_type: type[object] | None + supports_sync_iteration: bool | None + supports_async_iteration: bool | None + chunks: tuple[object, ...] + chunk_types: tuple[type[object], ...] + terminal: StreamTerminal + + +ChunkNormalizer: TypeAlias = Callable[[object], object] + + +def drain_sync_stream(stream: Iterable[object]) -> None: + for _ in stream: + pass + + +async def drain_async_stream(stream: AsyncIterable[object]) -> None: + async for _ in stream: + pass + + +def capture_sync_stream(create: Callable[[], Iterable[object]]) -> SDKReport: + return _stream_report(consume_sync_stream(create)) + + +async def capture_async_stream(create: Callable[[], Awaitable[AsyncIterable[object]]]) -> SDKReport: + return _stream_report(await consume_async_stream(create)) + + +def _stream_report(outcome: StreamOutcome) -> SDKReport: + terminal: Final = outcome.terminal + if isinstance(terminal, StreamFailed) and terminal.phase == "creation": + return terminal.error + return SDKStreamReport( + chunks=tuple(sdk_chunk(chunk) for chunk in outcome.chunks), + terminal=SDKStreamFailed(error=terminal.error) if isinstance(terminal, StreamFailed) else SDKStreamCompleted(), + ) + + +def _failed(phase: Literal["creation", "iteration"], error: Exception) -> StreamFailed: + return StreamFailed( + phase=phase, + exception_type=type(error), + error=sdk_error_report(error), + ) + + +def consume_sync_stream(create: Callable[[], Iterable[object]]) -> StreamOutcome: + try: + stream: Final = create() + except Exception as error: + return StreamOutcome( + wrapper_type=None, + supports_sync_iteration=None, + supports_async_iteration=None, + chunks=(), + chunk_types=(), + terminal=_failed("creation", error), + ) + + chunks: list[object] = [] # mutable-ok: iterator consumption builds an ordered trace + try: + for chunk in stream: + chunks.append(chunk) # noqa: PERF402 # partial trace is required if iteration raises + except Exception as error: + recorded: Final = tuple(chunks) + return StreamOutcome( + wrapper_type=type(stream), + supports_sync_iteration=hasattr(stream, "__iter__"), + supports_async_iteration=hasattr(stream, "__aiter__"), + chunks=recorded, + chunk_types=tuple(type(chunk) for chunk in recorded), + terminal=_failed("iteration", error), + ) + completed_chunks: Final = tuple(chunks) + return StreamOutcome( + wrapper_type=type(stream), + supports_sync_iteration=hasattr(stream, "__iter__"), + supports_async_iteration=hasattr(stream, "__aiter__"), + chunks=completed_chunks, + chunk_types=tuple(type(chunk) for chunk in completed_chunks), + terminal=StreamCompleted(), + ) + + +async def consume_async_stream(create: Callable[[], Awaitable[AsyncIterable[object]]]) -> StreamOutcome: + try: + stream: Final = await create() + except Exception as error: + return StreamOutcome( + wrapper_type=None, + supports_sync_iteration=None, + supports_async_iteration=None, + chunks=(), + chunk_types=(), + terminal=_failed("creation", error), + ) + + chunks: list[object] = [] # mutable-ok: iterator consumption builds an ordered trace + try: + async for chunk in stream: + chunks.append(chunk) + except Exception as error: + recorded: Final = tuple(chunks) + return StreamOutcome( + wrapper_type=type(stream), + supports_sync_iteration=hasattr(stream, "__iter__"), + supports_async_iteration=hasattr(stream, "__aiter__"), + chunks=recorded, + chunk_types=tuple(type(chunk) for chunk in recorded), + terminal=_failed("iteration", error), + ) + completed_chunks: Final = tuple(chunks) + return StreamOutcome( + wrapper_type=type(stream), + supports_sync_iteration=hasattr(stream, "__iter__"), + supports_async_iteration=hasattr(stream, "__aiter__"), + chunks=completed_chunks, + chunk_types=tuple(type(chunk) for chunk in completed_chunks), + terminal=StreamCompleted(), + ) + + +def normalize_chunk(chunk: object) -> object: + return chunk + + +def assert_stream_parity( + baseline: StreamOutcome, + candidate: StreamOutcome, + *, + normalize: ChunkNormalizer = normalize_chunk, +) -> None: + assert baseline.wrapper_type is candidate.wrapper_type + assert baseline.supports_sync_iteration is candidate.supports_sync_iteration + assert baseline.supports_async_iteration is candidate.supports_async_iteration + assert baseline.chunk_types == candidate.chunk_types + assert len(baseline.chunks) == len(candidate.chunks) + for index, (baseline_chunk, candidate_chunk) in enumerate(zip(baseline.chunks, candidate.chunks, strict=True)): + assert_value_parity(normalize(baseline_chunk), normalize(candidate_chunk), path=f"$.chunks[{index}]") + assert baseline.terminal == candidate.terminal diff --git a/tests/rust-python-harness/shared/parity/test_parity.py b/tests/rust-python-harness/shared/parity/test_parity.py new file mode 100644 index 00000000000..83daccdf8ba --- /dev/null +++ b/tests/rust-python-harness/shared/parity/test_parity.py @@ -0,0 +1,190 @@ +from __future__ import annotations + +from types import MappingProxyType +from typing import Final + +import pytest +from pydantic import BaseModel, ConfigDict, JsonValue, PrivateAttr + +from .compare import assert_model_parity, assert_parity +from .models import CapturedRequest, Execution, SDKError, SDKSuccess, sdk_error_report + +SENTINEL: Final = "python-parity-fallback" + + +class _ComparableResponse(BaseModel): + value: str + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + + def set_hidden_param(self, key: str, value: object) -> None: + self._hidden_params[key] = value + + +class _DifferentResponse(BaseModel): + value: str + + +class _FloatResponse(BaseModel): + values: list[float] + + +class _PublicValue(BaseModel): + model_config = ConfigDict(extra="allow") + + value: object + + +class _PublicError(ValueError): + status_code: Final = 400 + + +def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: str | None = None) -> Execution: + return Execution( + requests=( + CapturedRequest( + method="POST", + path="/v1/test-route?mode=test", + headers=(("authorization", "Bearer test-key"), ("content-type", "application/json")), + body={"model": "test-model"} if body is None else body, + user_agent=user_agent, + ), + ), + report=SDKSuccess(response={"items": [{"text": markdown}], "model": "test-model"}), + ) + + +def test_parity_rejects_request_difference() -> None: + python: Final = _execution(user_agent=SENTINEL) + rust: Final = _execution(body={"model": "different"}, user_agent="litellm-rust") + + with pytest.raises(AssertionError): + assert_parity(python, rust, SENTINEL) + + +def test_parity_rejects_response_difference() -> None: + python: Final = _execution(user_agent=SENTINEL) + rust: Final = _execution(markdown="different", user_agent="litellm-rust") + + with pytest.raises(AssertionError): + assert_parity(python, rust, SENTINEL) + + +def test_parity_rejects_error_difference() -> None: + python: Final = Execution( + requests=(), + report=SDKError( + exception_type="litellm.exceptions.BadRequestError", + message="bad request", + status_code=400, + code=None, + error_type=None, + param=None, + model="test-model", + llm_provider="test-provider", + ), + ) + rust: Final = python.model_copy(update={"report": python.report.model_copy(update={"status_code": 500})}) + + with pytest.raises(AssertionError): + assert_parity(python, rust, SENTINEL) + + +def test_sdk_error_report_removes_traceback_but_keeps_public_fields() -> None: + error: Final = _PublicError("invalid input\nTraceback (most recent call last):\n unstable") + + report: Final = sdk_error_report(error) + + assert report.exception_type.endswith("._PublicError") + assert report.message == "invalid input" + assert report.status_code == 400 + + +def test_parity_rejects_rust_fallback() -> None: + python: Final = _execution(user_agent=SENTINEL) + rust: Final = _execution(user_agent=SENTINEL) + + with pytest.raises(AssertionError, match="fell back"): + assert_parity(python, rust, SENTINEL) + + +def test_model_parity_compares_public_values_and_ignores_private_attrs() -> None: + python: Final = _ComparableResponse(value="same") + rust: Final = _ComparableResponse(value="same") + python.set_hidden_param("litellm_call_id", "python-id") + rust.set_hidden_param("litellm_call_id", "rust-id") + + assert_model_parity(python, rust) + + +def test_model_parity_rejects_public_value_difference() -> None: + python: Final = _ComparableResponse(value="python") + rust: Final = _ComparableResponse(value="rust") + + with pytest.raises(AssertionError): + assert_model_parity(python, rust) + + +def test_model_parity_rejects_type_difference() -> None: + with pytest.raises(AssertionError): + assert_model_parity(_ComparableResponse(value="same"), _DifferentResponse(value="same")) + + +def test_model_parity_rejects_wire_float_rounding_difference() -> None: + with pytest.raises(AssertionError, match=r"\$\.values\[0\]"): + assert_model_parity( + _FloatResponse(values=[0.22590550796036835]), + _FloatResponse(values=[0.22590550796036837]), + ) + + +def test_model_parity_rejects_meaningful_float_difference() -> None: + with pytest.raises(AssertionError, match=r"\$\.values\[0\]"): + assert_model_parity( + _FloatResponse(values=[0.22590550796036835]), + _FloatResponse(values=[0.2259]), + ) + + +@pytest.mark.parametrize( + ("baseline", "candidate"), + ( + (_ComparableResponse(value="same"), {"value": "same"}), + (_ComparableResponse(value="same"), _DifferentResponse(value="same")), + (True, 1), + (1, 1.0), + (["same"], ("same",)), + ({"value": "same"}, MappingProxyType({"value": "same"})), + ({True: "same"}, {1: "same"}), + ), + ids=("model-dict", "model-class", "bool-int", "int-float", "list-tuple", "mapping-class", "key-type"), +) +def test_model_parity_rejects_nested_type_changes(baseline: object, candidate: object) -> None: + with pytest.raises(AssertionError, match=r"\$\.value\[0\]"): + assert_model_parity(_PublicValue(value=[baseline]), _PublicValue(value=[candidate])) + + +def test_model_parity_ignores_nested_private_attributes() -> None: + baseline: Final = _ComparableResponse(value="same") + candidate: Final = _ComparableResponse(value="same") + baseline.set_hidden_param("request_id", "baseline") + candidate.set_hidden_param("request_id", "candidate") + + assert_model_parity(_PublicValue(value={"nested": [baseline]}), _PublicValue(value={"nested": [candidate]})) + + +@pytest.mark.parametrize("extras", ({"provider_value": "changed"}, {}, {"provider_value": {"value": "same"}})) +def test_model_parity_compares_public_extras(extras: dict[str, object]) -> None: + baseline: Final = _PublicValue.model_validate({"value": None, "provider_value": _ComparableResponse(value="same")}) + candidate: Final = _PublicValue.model_validate({"value": None, **extras}) + + with pytest.raises(AssertionError): + assert_model_parity(baseline, candidate) + + +def test_serialized_parity_rejects_boolean_integer_substitution() -> None: + with pytest.raises(AssertionError, match="type mismatch"): + assert_parity( + _execution(body={"enabled": True}, user_agent=SENTINEL), + _execution(body={"enabled": 1}, user_agent="candidate"), + SENTINEL, + ) diff --git a/tests/rust-python-harness/shared/parity/test_stream.py b/tests/rust-python-harness/shared/parity/test_stream.py new file mode 100644 index 00000000000..48d3cec0407 --- /dev/null +++ b/tests/rust-python-harness/shared/parity/test_stream.py @@ -0,0 +1,344 @@ +from __future__ import annotations + +import queue +from collections.abc import AsyncIterator, Iterator +from typing import Final, Literal, NoReturn + +import pytest +from pydantic import BaseModel, PrivateAttr, ValidationError + +from .models import ( + SDKBytesChunk, + SDKError, + SDKJsonChunk, + SDKReport, + SDKStreamCompleted, + SDKStreamFailed, + SDKStreamReport, + sdk_error_report, +) +from .stream import ( + StreamCompleted, + StreamFailed, + StreamOutcome, + assert_stream_parity, + capture_async_stream, + capture_sync_stream, + consume_async_stream, + consume_sync_stream, + drain_async_stream, + drain_sync_stream, +) + + +class _Chunk(BaseModel): + value: str + _hidden_params: dict[str, object] = PrivateAttr(default_factory=dict) + + def set_hidden_param(self, key: str, value: object) -> None: + self._hidden_params[key] = value + + +class _NestedChunk(BaseModel): + value: object + + +class _SyncStream: + def __init__(self, chunks: tuple[object, ...], error: BaseException | None = None) -> None: + self.chunks: Final = chunks + self.error: Final = error + + def __iter__(self) -> Iterator[object]: + yield from self.chunks + if self.error is not None: + raise self.error + + +class _AsyncStream: + def __init__(self, chunks: tuple[object, ...], error: BaseException | None = None) -> None: + self.chunks: Final = chunks + self.error: Final = error + + async def __aiter__(self) -> AsyncIterator[object]: + for chunk in self.chunks: + yield chunk + if self.error is not None: + raise self.error + + +class _PublicStreamError(Exception): + def __init__( + self, + message: str = "invalid input", + *, + status_code: int = 400, + llm_provider: str = "test", + model: str = "test-model", + code: str = "invalid_input", + error_type: str = "validation_error", + param: str = "input", + ) -> None: + super().__init__(message) + self.status_code: Final = status_code + self.llm_provider: Final = llm_provider + self.model: Final = model + self.code: Final = code + self.type: Final = error_type + self.param: Final = param + + +def _creation_error() -> NoReturn: + raise _PublicStreamError(status_code=429, llm_provider="test", model="test-model") + + +async def _async_stream(chunks: tuple[object, ...], error: BaseException | None = None) -> _AsyncStream: + return _AsyncStream(chunks, error) + + +async def _consume( + mode: Literal["sync", "async"], chunks: tuple[object, ...], error: Exception | None = None +) -> StreamOutcome: + if mode == "sync": + return consume_sync_stream(lambda: _SyncStream(chunks, error)) + return await consume_async_stream(lambda: _async_stream(chunks, error)) + + +async def _capture( + mode: Literal["sync", "async"], chunks: tuple[object, ...], error: Exception | None = None +) -> SDKReport: + if mode == "sync": + return capture_sync_stream(lambda: _SyncStream(chunks, error)) + return await capture_async_stream(lambda: _async_stream(chunks, error)) + + +def test_sync_stream_parity_compares_chunks_and_ignores_private_attrs() -> None: + python_chunk: Final = _Chunk(value="same") + accelerated_chunk: Final = _Chunk(value="same") + python_chunk.set_hidden_param("request_id", "python") + accelerated_chunk.set_hidden_param("request_id", "accelerated") + python: Final = consume_sync_stream(lambda: _SyncStream((python_chunk,))) + accelerated: Final = consume_sync_stream(lambda: _SyncStream((accelerated_chunk,))) + + assert python.supports_sync_iteration is True + assert python.supports_async_iteration is False + assert_stream_parity(python, accelerated) + + +def test_stream_parity_rejects_extra_chunk() -> None: + python: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="one"),))) + accelerated: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="one"), _Chunk(value="two")))) + + with pytest.raises(AssertionError): + assert_stream_parity(python, accelerated) + + +def test_stream_parity_rejects_chunk_value_difference() -> None: + python: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="python"),))) + accelerated: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="accelerated"),))) + + with pytest.raises(AssertionError): + assert_stream_parity(python, accelerated) + + +def test_stream_outcome_distinguishes_creation_and_iteration_errors() -> None: + creation: Final = consume_sync_stream(_creation_error) + iteration: Final = consume_sync_stream( + lambda: _SyncStream( + (_Chunk(value="before-error"),), + _PublicStreamError(status_code=429, llm_provider="test", model="test-model"), + ) + ) + + assert isinstance(creation.terminal, StreamFailed) + assert creation.terminal.phase == "creation" + assert creation.chunks == () + assert isinstance(iteration.terminal, StreamFailed) + assert iteration.terminal.phase == "iteration" + assert len(iteration.chunks) == 1 + with pytest.raises(AssertionError): + assert_stream_parity(creation, iteration) + + +@pytest.mark.asyncio +async def test_async_stream_uses_same_trace_contract() -> None: + python_error: Final = _PublicStreamError( + "invalid input\nTraceback (most recent call last):\npython detail", status_code=500 + ) + accelerated_error: Final = _PublicStreamError( + "invalid input\nTraceback (most recent call last):\nrust detail", status_code=500 + ) + python: Final = await consume_async_stream(lambda: _async_stream((_Chunk(value="same"),), python_error)) + accelerated: Final = await consume_async_stream(lambda: _async_stream((_Chunk(value="same"),), accelerated_error)) + + assert python.supports_sync_iteration is False + assert python.supports_async_iteration is True + assert_stream_parity(python, accelerated) + + +def test_stream_parity_accepts_route_specific_chunk_normalizer() -> None: + python: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="python-generated-id"),))) + accelerated: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="rust-generated-id"),))) + + assert_stream_parity(python, accelerated, normalize=lambda chunk: type(chunk)) + + +def test_drain_sync_stream_exhausts_lazy_iterator() -> None: + consumed: Final[queue.SimpleQueue[str]] = queue.SimpleQueue() + + def chunks() -> Iterator[object]: + yield _Chunk(value="one") + consumed.put("complete") + + drain_sync_stream(chunks()) + + assert consumed.get_nowait() == "complete" + + +@pytest.mark.asyncio +async def test_drain_async_stream_exhausts_lazy_iterator() -> None: + consumed: Final[queue.SimpleQueue[str]] = queue.SimpleQueue() + + async def chunks() -> AsyncIterator[object]: + yield b"one" + consumed.put("complete") + + await drain_async_stream(chunks()) + + assert consumed.get_nowait() == "complete" + + +def test_capture_sync_stream_serializes_model_chunks_and_partial_failure() -> None: + report: Final = capture_sync_stream( + lambda: _SyncStream( + (_Chunk(value="before-error"),), + _PublicStreamError(status_code=429, llm_provider="test", model="test-model"), + ) + ) + + assert isinstance(report, SDKStreamReport) + assert len(report.chunks) == 1 + chunk: Final = report.chunks[0] + assert isinstance(chunk, SDKJsonChunk) + assert chunk.value == {"value": "before-error"} + assert isinstance(report.terminal, SDKStreamFailed) + assert report.terminal.error.status_code == 429 + + +@pytest.mark.asyncio +async def test_capture_async_stream_serializes_message_bytes_in_order() -> None: + report: Final = await capture_async_stream(lambda: _async_stream((b"first", b"second"))) + + assert isinstance(report, SDKStreamReport) + assert tuple(chunk.data_bytes() for chunk in report.chunks if isinstance(chunk, SDKBytesChunk)) == ( + b"first", + b"second", + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize( + "candidate_error", + ( + ValueError("invalid input"), + _PublicStreamError("changed message"), + _PublicStreamError(status_code=429), + _PublicStreamError(code="changed_code"), + _PublicStreamError(error_type="changed_type"), + _PublicStreamError(param="changed_param"), + _PublicStreamError(model="changed_model"), + _PublicStreamError(llm_provider="changed_provider"), + ), + ids=("exception", "message", "status", "code", "type", "param", "model", "provider"), +) +async def test_stream_parity_rejects_public_error_changes( + mode: Literal["sync", "async"], candidate_error: Exception +) -> None: + chunks: Final = (_Chunk(value="partial"),) + baseline: Final = await _consume(mode, chunks, _PublicStreamError()) + candidate: Final = await _consume(mode, chunks, candidate_error) + + with pytest.raises(AssertionError): + assert_stream_parity(baseline, candidate) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize("candidate", (("one",), ("one", "two", "three"), ("two", "one"), ("one", "changed"))) +async def test_stream_parity_checks_event_sequence(mode: Literal["sync", "async"], candidate: tuple[str, ...]) -> None: + baseline: Final = await _consume(mode, (_Chunk(value="one"), _Chunk(value="two"))) + changed: Final = await _consume(mode, tuple(_Chunk(value=value) for value in candidate)) + + with pytest.raises(AssertionError): + assert_stream_parity(baseline, changed) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_stream_parity_preserves_nested_types_and_ignores_private_fields(mode: Literal["sync", "async"]) -> None: + first: Final = _Chunk(value="same") + second: Final = _Chunk(value="same") + first.set_hidden_param("request_id", "first") + second.set_hidden_param("request_id", "second") + baseline: Final = await _consume(mode, (_NestedChunk(value=[first]),)) + candidate: Final = await _consume(mode, (_NestedChunk(value=[second]),)) + + assert_stream_parity(baseline, candidate) + changed: Final = await _consume(mode, (_NestedChunk(value=[{"value": "same"}]),)) + with pytest.raises(AssertionError, match=r"\$\.chunks\[0\]\.value\[0\]"): + assert_stream_parity(baseline, changed) + + +def test_stream_parity_rejects_wrapper_and_chunk_type_changes() -> None: + baseline: Final = consume_sync_stream(lambda: _SyncStream((_Chunk(value="same"),))) + different_wrapper: Final = consume_sync_stream(lambda: iter((_Chunk(value="same"),))) + different_chunk: Final = consume_sync_stream(lambda: _SyncStream(({"value": "same"},))) + + with pytest.raises(AssertionError): + assert_stream_parity(baseline, different_wrapper) + with pytest.raises(AssertionError): + assert_stream_parity(baseline, different_chunk) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +@pytest.mark.parametrize("error", (None, _PublicStreamError())) +async def test_capture_keeps_serialization_failures_out_of_sdk_errors( + mode: Literal["sync", "async"], error: Exception | None +) -> None: + with pytest.raises(ValidationError): + await _capture(mode, (_Chunk(value="valid"), object()), error) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_empty_stream_completes(mode: Literal["sync", "async"]) -> None: + outcome: Final = await _consume(mode, ()) + assert outcome.chunks == () + assert outcome.terminal == StreamCompleted() + assert await _capture(mode, ()) == SDKStreamReport(chunks=(), terminal=SDKStreamCompleted()) + + +@pytest.mark.asyncio +async def test_async_creation_error_matches_sync_capture() -> None: + async def create() -> _AsyncStream: + _creation_error() + + sync: Final = consume_sync_stream(_creation_error) + asynchronous: Final = await consume_async_stream(create) + assert_stream_parity(sync, asynchronous) + assert isinstance(sync.terminal, StreamFailed) + assert sync.terminal.phase == "creation" + assert isinstance(capture_sync_stream(_creation_error), SDKError) + assert capture_sync_stream(_creation_error) == await capture_async_stream(create) == sync.terminal.error + + +@pytest.mark.asyncio +@pytest.mark.parametrize("mode", ("sync", "async")) +async def test_capture_preserves_partial_output_and_complete_error(mode: Literal["sync", "async"]) -> None: + error: Final = _PublicStreamError() + report: Final = await _capture(mode, (_Chunk(value="partial"),), error) + assert report == SDKStreamReport( + chunks=(SDKJsonChunk(value={"value": "partial"}),), + terminal=SDKStreamFailed(error=sdk_error_report(error)), + ) diff --git a/tests/rust-python-harness/shared/reporting/__init__.py b/tests/rust-python-harness/shared/reporting/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/models.py b/tests/rust-python-harness/shared/reporting/models.py similarity index 91% rename from tests/rust-python-harness/models.py rename to tests/rust-python-harness/shared/reporting/models.py index a02684dc63d..4ffacdce9ab 100644 --- a/tests/rust-python-harness/models.py +++ b/tests/rust-python-harness/shared/reporting/models.py @@ -44,10 +44,12 @@ class HarnessCase: coverage: Coverage selectors: tuple[str, ...] note: str = "" + surface: str = "sdk" + unit_suite: str | None = None @property def key(self) -> str: - return f"{self.strategy_id}:{self.sdk_function}" + return f"{self.strategy_id}:{self.sdk_function}" if self.surface == "sdk" else f"{self.strategy_id}:gateway:{self.sdk_function}" @dataclass(frozen=True) @@ -96,7 +98,7 @@ class CaseResult: def set_initial_status(self) -> None: if self.case.coverage is Coverage.NOT_APPLICABLE: self.status = RunStatus.NOT_APPLICABLE - elif not self.case.selectors: + elif not self.case.selectors and not self.case.unit_suite: self.status = RunStatus.PLANNED else: self.status = RunStatus.QUEUED @@ -168,12 +170,13 @@ def section_confidence( ) -> tuple[SectionConfidence, ...]: strategy_list = tuple(strategies) scores: list[SectionConfidence] = [] - for sdk_function in SDK_FUNCTIONS: + sections = tuple(dict.fromkeys((case.surface, case.sdk_function) for strategy in strategy_list for case in strategy.cases)) + for surface, sdk_function in sections: cases = tuple( case for strategy in strategy_list for case in strategy.cases - if case.sdk_function == sdk_function + if case.sdk_function == sdk_function and case.surface == surface and case.coverage is not Coverage.NOT_APPLICABLE ) verified = 0 @@ -195,7 +198,7 @@ def section_confidence( level = ConfidenceLevel.LOW scores.append( SectionConfidence( - sdk_function=sdk_function, + sdk_function=sdk_function if surface == "sdk" else f"gateway/{sdk_function}", verified_strategies=verified, required_strategies=required, level=level, diff --git a/tests/rust-python-harness/shared/reporting/orchestration.py b/tests/rust-python-harness/shared/reporting/orchestration.py new file mode 100644 index 00000000000..5e1c0ff57d8 --- /dev/null +++ b/tests/rust-python-harness/shared/reporting/orchestration.py @@ -0,0 +1,65 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from pathlib import Path +from time import monotonic +from typing import Final, Protocol + +from .models import HarnessCase, HarnessRun +from .pytest_runner import UpdateCallback + + +class StrategyRunner(Protocol): + def __call__( + self, + cases: Sequence[HarnessCase], + repo_root: Path, + on_update: UpdateCallback, + pytest_args: Sequence[str] = (), + ) -> tuple[int, HarnessRun]: ... + + +def combine_reports(reports: Sequence[HarnessRun]) -> HarnessRun: + return HarnessRun( + results={key: result for report in reports for key, result in report.results.items()}, + current_nodeid=next((report.current_nodeid for report in reversed(reports) if report.current_nodeid), None), + failures=[failure for report in reports for failure in report.failures], + started_at=min((report.started_at for report in reports), default=monotonic()), + finished_at=( + max((report.finished_at for report in reports if report.finished_at is not None), default=None) + if all(report.finished_at is not None for report in reports) + else None + ), + ) + + +def run_strategies( + cases: Sequence[HarnessCase], + repo_root: Path, + on_update: UpdateCallback, + pytest_args: Sequence[str], + resolve_runner: Callable[[str], StrategyRunner], +) -> tuple[int, HarnessRun]: + strategy_ids: Final = tuple(dict.fromkeys(case.strategy_id for case in cases)) + + def execute( + remaining: tuple[str, ...], reports: tuple[HarnessRun, ...], codes: tuple[int, ...] + ) -> tuple[int, HarnessRun]: + if not remaining: + combined: Final = combine_reports(reports) + on_update(combined) + return next((code for code in codes if code), 0), combined + strategy_id, *tail = remaining + selected: Final = tuple(case for case in cases if case.strategy_id == strategy_id) + pending: Final = HarnessRun.from_cases(case for case in cases if case.strategy_id in tail) + code, report = resolve_runner(strategy_id)( + selected, + repo_root, + lambda current: on_update(combine_reports((*reports, current, pending))), + pytest_args, + ) + if code in {2, 3, 4}: + return code, combine_reports((*reports, report, pending)) + return execute(tuple(tail), (*reports, report), (*codes, code)) + + return execute(strategy_ids, (), ()) diff --git a/tests/rust-python-harness/runner.py b/tests/rust-python-harness/shared/reporting/pytest_runner.py similarity index 89% rename from tests/rust-python-harness/runner.py rename to tests/rust-python-harness/shared/reporting/pytest_runner.py index 286f2c4116d..a7e73308f30 100644 --- a/tests/rust-python-harness/runner.py +++ b/tests/rust-python-harness/shared/reporting/pytest_runner.py @@ -4,6 +4,7 @@ import os from collections.abc import Callable, Sequence from pathlib import Path from time import monotonic +from typing import Final import pytest @@ -148,13 +149,22 @@ def run_pytest( return exit_code, run plugin = HarnessPytestPlugin(run=run, on_update=on_update) - args = [*selectors, "-p", "no:terminal", *pytest_args] + args: Final = (*selectors, "-q", "--tb=no", "--no-summary", "-o", "consider_namespace_packages=true", *pytest_args) previous_directory = Path.cwd() try: os.chdir(repo_root) - exit_code = int(pytest.main(args, plugins=[plugin])) + exit_code = int(pytest.main(list(args), plugins=[plugin])) finally: os.chdir(previous_directory) + for result in run.results.values(): + missing = tuple( + selector for selector in result.case.selectors + if not any(selector_matches_node(selector, node) for node in result.collected) + ) + if missing: + result.status = RunStatus.MISSING + run.failures.extend((selector, "Configured selector collected no tests") for selector in missing) + on_update(run) if exit_code == 0 and any( result.status is RunStatus.MISSING for result in run.results.values() ): diff --git a/tests/rust-python-harness/shared/reporting/test_orchestration.py b/tests/rust-python-harness/shared/reporting/test_orchestration.py new file mode 100644 index 00000000000..8aea6d67e6e --- /dev/null +++ b/tests/rust-python-harness/shared/reporting/test_orchestration.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Final + +from .models import Coverage, HarnessCase, RunStatus +from .orchestration import run_strategies +from .pytest_runner import run_pytest + + +def test_combines_independent_strategy_reports_and_keeps_failures(tmp_path: Path) -> None: + (tmp_path / "test_first.py").write_text("def test_first():\n assert 1 == 2\n") + (tmp_path / "test_second.py").write_text("def test_second():\n assert True\n") + cases: Final = tuple( + HarnessCase( + strategy_id=name, + strategy_label=name, + sdk_function="ocr", + coverage=Coverage.COMPLETE, + selectors=(f"test_{name}.py",), + ) + for name in ("first", "second") + ) + code, report = run_strategies(cases, tmp_path, lambda _: None, (), lambda _: run_pytest) + assert code == 1 + assert report.results["first:ocr"].status is RunStatus.FAILED + assert report.results["second:ocr"].status is RunStatus.PASSED + assert report.completed_tests == 2 + assert len(report.failures) == 1 + assert "assert 1 == 2" in report.failures[0][1] + assert "terminalreporter" not in report.failures[0][1] + + +def test_missing_selector_cannot_hide_behind_a_passing_surface(tmp_path: Path) -> None: + (tmp_path / "test_present.py").write_text("def test_present():\n assert True\n") + case: Final = HarnessCase( + strategy_id="e2e_parity", + strategy_label="End-to-end parity", + sdk_function="ocr", + surface="gateway", + coverage=Coverage.PARTIAL, + selectors=("test_present.py", "test_missing.py"), + ) + code, report = run_pytest((case,), tmp_path, lambda _: None) + assert code == 1 + assert report.results["e2e_parity:gateway:ocr"].status is RunStatus.MISSING + assert ("test_missing.py", "Configured selector collected no tests") in report.failures diff --git a/tests/rust-python-harness/ui.py b/tests/rust-python-harness/shared/reporting/ui.py similarity index 89% rename from tests/rust-python-harness/ui.py rename to tests/rust-python-harness/shared/reporting/ui.py index bf143a4e100..3807af8c53b 100644 --- a/tests/rust-python-harness/ui.py +++ b/tests/rust-python-harness/shared/reporting/ui.py @@ -12,7 +12,6 @@ from .models import ( Coverage, HarnessRun, RunStatus, - SDK_FUNCTIONS, Strategy, section_confidence, ) @@ -52,7 +51,9 @@ def _format_duration(seconds: float) -> str: def _rerun_command(nodeid: str) -> str: - return f"poetry run pytest {shlex.quote(nodeid)} -q" + if nodeid.startswith("unit-suite:"): + return "uv run python -m tests.rust-python-harness.strategies.unit_tests.runner --plain" + return f"poetry run pytest {shlex.quote(nodeid)} -q -o consider_namespace_packages=true" def _summary(run: HarnessRun) -> tuple[int, int, int, int]: @@ -67,8 +68,9 @@ def _summary(run: HarnessRun) -> tuple[int, int, int, int]: ) -def _cell_text(run: HarnessRun, strategy_id: str, sdk_function: str) -> tuple[str, str]: - result = run.results.get(f"{strategy_id}:{sdk_function}") +def _cell_text(run: HarnessRun, strategy_id: str, sdk_function: str, surface: str = "sdk") -> tuple[str, str]: + key = f"{strategy_id}:{sdk_function}" if surface == "sdk" else f"{strategy_id}:gateway:{sdk_function}" + result = run.results.get(key) if result is None: return "", "" counts = "" @@ -101,6 +103,7 @@ class RichDashboard(AbstractContextManager["RichDashboard"]): from rich.table import Table from rich.text import Text + columns = tuple(dict.fromkeys((case.surface, case.sdk_function) for strategy in self.strategies for case in strategy.cases)) narrow = self.console.width < 96 if narrow: table = Table(box=box.SIMPLE_HEAVY, expand=True, show_header=False) @@ -108,23 +111,23 @@ class RichDashboard(AbstractContextManager["RichDashboard"]): table.add_column("Results", ratio=5) for strategy in self.strategies: values = [] - for sdk_function in SDK_FUNCTIONS: - value, style = _cell_text(run, strategy.id, sdk_function) + for surface, sdk_function in columns: + value, style = _cell_text(run, strategy.id, sdk_function, surface) if value: values.append( - Text.assemble((f"{sdk_function} ", "dim"), (value, style)) + Text.assemble((f"{surface}/{sdk_function} ", "dim"), (value, style)) ) table.add_row(strategy.label, Text(" ").join(values)) return table - table = Table(box=box.ROUNDED, expand=True, title="Strategy × SDK function") + table = Table(box=box.ROUNDED, expand=True, title="Strategy × API") table.add_column("Strategy", ratio=3) - for label in SDK_FUNCTIONS: - table.add_column(label, justify="center", ratio=1) + for surface, label in columns: + table.add_column(label if surface == "sdk" else f"gateway/{label}", justify="center", ratio=1) for strategy in self.strategies: cells = [] - for sdk_function in SDK_FUNCTIONS: - value, style = _cell_text(run, strategy.id, sdk_function) + for surface, sdk_function in columns: + value, style = _cell_text(run, strategy.id, sdk_function, surface) cells.append(Text(value, style=style)) table.add_row(strategy.label, *cells) return table @@ -192,7 +195,7 @@ class RichDashboard(AbstractContextManager["RichDashboard"]): from rich.table import Table confidence_table = Table( - title="Port confidence by SDK section", box=box.ROUNDED, expand=True + title="Port confidence by API", box=box.ROUNDED, expand=True ) confidence_table.add_column("SDK section") confidence_table.add_column("Score", justify="right") @@ -256,9 +259,10 @@ class PlainDashboard(AbstractContextManager["PlainDashboard"]): f"{skipped} skipped in {_format_duration(run.duration)}", flush=True, ) - for nodeid, _ in run.failures[:5]: + for nodeid, detail in run.failures[:5]: + print(f"{nodeid}: {detail}", flush=True) print(f"Rerun: {_rerun_command(nodeid)}", flush=True) - print("Port confidence by SDK section", flush=True) + print("Port confidence by API", flush=True) for score in section_confidence(run, self.confidence_strategies): print( f" {score.sdk_function:12} " diff --git a/tests/rust-python-harness/shared/tracing/__init__.py b/tests/rust-python-harness/shared/tracing/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/shared/tracing/compare.py b/tests/rust-python-harness/shared/tracing/compare.py new file mode 100644 index 00000000000..9c43bea6c0e --- /dev/null +++ b/tests/rust-python-harness/shared/tracing/compare.py @@ -0,0 +1,57 @@ +from __future__ import annotations + +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from typing import Final + + +@dataclass(frozen=True, slots=True) +class Operation: + name: str + started: int + finished: int + + +def compare_traces( + python: Sequence[Operation], + rust: Sequence[Operation], + mapping: Mapping[str, str], + required_order: Sequence[tuple[str, str]] = (), +) -> tuple[str, ...]: + python_names: Final = {operation.name for operation in python} + rust_names: Final = {operation.name for operation in rust} + problems: Final = ( + *(f"unmapped Python operation: {name}" for name in sorted(python_names - mapping.keys())), + *(f"unmapped Rust operation: {name}" for name in sorted(rust_names - set(mapping.values()))), + *(f"ambiguous Rust operation: {name}" for name, count in Counter(mapping.values()).items() if count > 1), + *( + f"invalid interval: {operation.name}" + for operation in (*python, *rust) + if operation.started > operation.finished + ), + ) + if problems: + return problems + python_counts: Final = Counter(operation.name for operation in python) + rust_counts: Final = Counter(operation.name for operation in rust) + counts: Final = tuple( + f"call count differs for {name}: Python={python_counts[name]}, Rust={rust_counts[target]}" + for name, target in mapping.items() + if python_counts[name] != rust_counts[target] + ) + ordering: Final = tuple( + f"{label}: required order {before} before {after} was not observed" + for before, after in required_order + for label, operations, first, second in ( + ("Python", python, before, after), + ("Rust", rust, mapping.get(before), mapping.get(after)), + ) + if not first + or not second + or not any(operation.name == first for operation in operations) + or not any(operation.name == second for operation in operations) + or max(operation.finished for operation in operations if operation.name == first) + > min(operation.started for operation in operations if operation.name == second) + ) + return (*counts, *ordering) diff --git a/tests/rust-python-harness/shared/tracing/test_compare.py b/tests/rust-python-harness/shared/tracing/test_compare.py new file mode 100644 index 00000000000..2dfad24846b --- /dev/null +++ b/tests/rust-python-harness/shared/tracing/test_compare.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import pytest + +from .compare import Operation, compare_traces + + +@pytest.mark.parametrize( + ("rust", "message"), + ( + ((Operation("decode", 0, 1), Operation("send", 2, 3)), None), + ((Operation("decode", 0, 1), Operation("send", 2, 3), Operation("send", 4, 5)), "call count differs"), + ((Operation("send", 0, 1), Operation("decode", 2, 3)), "required order"), + ((Operation("decode", 0, 4), Operation("send", 2, 3)), "required order"), + ((Operation("decode", 0, 1), Operation("unknown", 2, 3)), "unmapped Rust"), + ), +) +def test_compare_mapped_calls_and_required_completion_order(rust: tuple[Operation, ...], message: str | None) -> None: + problems = compare_traces( + (Operation("parse", 0, 1), Operation("request", 2, 3)), + rust, + {"parse": "decode", "request": "send"}, + (("parse", "request"),), + ) + if message is None: + assert problems == () + else: + assert any(message in problem for problem in problems) + + +def test_missing_required_operations_and_ambiguous_mappings_fail() -> None: + assert compare_traces((), (), {"parse": "decode"}, (("parse", "request"),)) + assert compare_traces((), (), {"parse": "decode", "request": "decode"}) == ("ambiguous Rust operation: decode",) diff --git a/tests/rust-python-harness/strategies/e2e_parity/README.md b/tests/rust-python-harness/strategies/e2e_parity/README.md new file mode 100644 index 00000000000..17643e69676 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/README.md @@ -0,0 +1,5 @@ +# E2E Parity + +Run independently with `uv run python -m tests.rust-python-harness.strategies.e2e_parity.runner --plain`. Configure SDK and gateway selectors in `strategy.json`; keep API-specific execution and fixtures in their owning surface folder + +See [the harness guide](../../README.md) for coverage status and shared comparison tools diff --git a/tests/rust-python-harness/strategies/e2e_parity/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/e2e_parity/gateway/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/gateway/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/e2e_parity/runner.py b/tests/rust-python-harness/strategies/e2e_parity/runner.py new file mode 100644 index 00000000000..2886c823370 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/runner.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from ...shared.reporting.models import HarnessCase, HarnessRun +from ...shared.reporting.pytest_runner import UpdateCallback, run_pytest + + +def run( + cases: Sequence[HarnessCase], + repo_root: Path, + on_update: UpdateCallback, + pytest_args: Sequence[str] = (), +) -> tuple[int, HarnessRun]: + return run_pytest(cases, repo_root, on_update, pytest_args) + + +def main(argv: Sequence[str] | None = None) -> int: + from ...cli import main as harness_main + + return harness_main(argv, strategy_id="e2e_parity") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/chat_completions/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/chat_completions/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/conftest.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/conftest.py new file mode 100644 index 00000000000..37d78a7df4c --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/conftest.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from typing import Final + +import pytest + +from .....shared.parity.fixtures.pytest_support import parametrize_recorded_fixtures +from .....shared.parity.fixtures.store import fixture_id +from .fixtures.config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV +from .fixtures.models import OcrParityCase + + +def ocr_fixture_id(fixture: OcrParityCase) -> str: + case_input: Final = fixture.litellm_input + provider: Final = case_input.custom_llm_provider + prefix: Final = f"{provider}/{case_input.model}" if provider else case_input.model + return fixture_id(case_input, prefix) + + +def ocr_fixture_marks(fixture: OcrParityCase) -> tuple[pytest.MarkDecorator, ...]: + if fixture.litellm_input.contract not in {"reducto_v3", "reducto_legacy"}: + return () + return ( + pytest.mark.xfail( + reason="Reducto does not have a Rust OCR contract", + strict=False, + ), + ) + + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + parametrize_recorded_fixtures( + metafunc, + fixture_name="ocr_fixture", + case_type=OcrParityCase, + env_var=FIXTURE_DIR_ENV, + default_directory=DEFAULT_FIXTURE_DIRECTORY, + regeneration_command=( + f"uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.record --fixture-dir {DEFAULT_FIXTURE_DIRECTORY}" + ), + id_builder=ocr_fixture_id, + marks_builder=ocr_fixture_marks, + ) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/README.md b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/README.md new file mode 100644 index 00000000000..0b4fbd2c6c8 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/README.md @@ -0,0 +1,49 @@ +# OCR parity fixtures + +The recording command runs four stages: + +1. Generate deterministic SDK inputs for every configured OCR target +2. Build target-scoped, deduplicated recording jobs +3. Record upstream responses through one globally bounded worker pool +4. Persist each fixture and report whether it was recorded, cached, or failed + +Run it with: + +```shell +uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.record --examples 4 --concurrency 4 +``` + +`--concurrency` caps provider calls across all targets. Independent jobs finish after a failure, then the command exits +nonzero if any job failed + +New recordings are VCR YAML cassettes. The committed corpus contains 31 migrated cassettes: 18 Mistral, 9 Reducto v3, +and 4 Reducto legacy. Their original response bytes, statuses, headers, and recording timestamps are preserved + +To migrate an existing JSON fixture directory locally: + +```shell +uv run python -m tests.rust-python-harness.strategies.e2e_parity.sdk.ocr.fixtures.migrate --fixture-dir tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data +``` + +The migration replays each old response through the Python SDK to reconstruct missing requests, writes and validates +the YAML cassette, then removes its JSON predecessor. It calls only local recording/replay servers and needs no provider +credentials. Reconstructed requests are labeled `python_replay`; they are not historical wire captures. Filenames use +the current normalized SDK input hash, including the fixture contract + +OCR strategies generate public `litellm.ocr()` and `litellm.aocr()` inputs. Every case contains the normalized model, +document, optional provider override, and LiteLLM keyword arguments. The fixture-only `contract` literal selects the +input schema and is removed before calling the SDK. Strategies never build provider wire payloads + +Each contract has a required corpus containing a baseline and cases for its supported top-level OCR parameters. The +contracts are Mistral, Azure-hosted Mistral, Vertex-hosted Mistral, Azure Document Intelligence, Vertex DeepSeek, +Reducto v3, and Reducto legacy. Credentials and endpoints only control target discovery, so a machine records the +contracts it has configured and skips the rest + +Reducto fixtures record upload and parse responses. Their parity cases remain non-strict expected failures until the +Rust OCR bridge supports Reducto. Azure and Vertex generation paths are unit-tested without credentials in CI, so the +committed corpus does not need live recordings for every target + +Every recording target owns a small fixed provider-rejected corpus, independent of replay implementation support. +Those inputs are recorded separately from generated valid inputs. Local validation failures use no recorded response; +the parity suite checks those unsupported providers and models, malformed documents, invalid request formats, invalid +Azure Document Intelligence parameters, and invalid headers in sync and async SDK calls diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/azure.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/azure.py new file mode 100644 index 00000000000..58487c2254c --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/azure.py @@ -0,0 +1,207 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final, Literal, cast + +from hypothesis import strategies as st +from hypothesis.strategies import SearchStrategy +from pydantic import StrictInt, StrictStr, field_validator + +from ......shared.parity.fixtures.recording import UpstreamEndpoint +from .base import OcrDocument, OcrSdkInputBase +from .common import ( + OcrFixtureClient, + OcrRecordingTarget, + image_document, + invoke_with_api_key, + pdf_document, +) +from .mistral import ( + MistralCompatibleOcrSdkInput, + mistral_input_values_strategy, +) + +AzureMistralModel = Literal["azure_ai/mistral-document-ai-2512",] +AzureMistralFixtureModel = AzureMistralModel | Literal["azure_ai/invalid-ocr-model-for-parity"] +AzureDocumentIntelligenceModel = Literal[ + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", + "azure_ai/doc-intelligence/prebuilt-document", +] +AzureDocumentIntelligenceFixtureModel = ( + AzureDocumentIntelligenceModel | Literal["azure_ai/doc-intelligence/invalid-ocr-model-for-parity"] +) + +AZURE_MISTRAL_MODELS: Final[tuple[AzureMistralModel, ...]] = ("azure_ai/mistral-document-ai-2512",) +AZURE_DOCUMENT_INTELLIGENCE_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = ( + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", + "azure_ai/doc-intelligence/prebuilt-document", +) +# API v4 replaces prebuilt-document with prebuilt-layout plus keyValuePairs. Keep +# the broader fixture model above so existing recordings remain loadable. +AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = ( + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", +) + + +class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput): + contract: Literal["azure_mistral"] = "azure_mistral" + model: AzureMistralFixtureModel + custom_llm_provider: Literal["azure_ai"] | None = None + + @field_validator("model") + @classmethod + def validate_model_namespace(cls, model: str) -> str: + if not model.startswith("azure_ai/"): + raise ValueError("Azure Mistral models must use the azure_ai/ LiteLLM namespace") + return model + + +class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase): + contract: Literal["azure_document_intelligence"] = "azure_document_intelligence" + model: AzureDocumentIntelligenceFixtureModel + document: OcrDocument + custom_llm_provider: Literal["azure_ai"] | None = None + pages: str | list[StrictInt] | list[StrictStr] | None = None + features: str | list[str] | None = None + req_format: Literal["litellm"] = "litellm" + + +AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureMistralOcrSdkInput, ...]] = ( + AzureMistralOcrSdkInput( + model="azure_ai/invalid-ocr-model-for-parity", + document=pdf_document(), + ), +) +AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS: Final[tuple[AzureDocumentIntelligenceOcrSdkInput, ...]] = ( + AzureDocumentIntelligenceOcrSdkInput( + model="azure_ai/doc-intelligence/invalid-ocr-model-for-parity", + document=pdf_document(), + ), +) + + +def _azure_mistral_input(values: dict[str, object], model: AzureMistralModel) -> AzureMistralOcrSdkInput: + return AzureMistralOcrSdkInput.model_validate({**values, "model": model}) + + +def azure_mistral_input_strategy(inline_image_data_uri: str) -> SearchStrategy[AzureMistralOcrSdkInput]: + # Foundry's active gateway schema rejects 2512-only controls and + # document_annotation_prompt, even though native Mistral accepts them. + return st.builds( + _azure_mistral_input, + values=mistral_input_values_strategy("2505", inline_image_data_uri, include_document_annotation_prompt=False), + model=st.sampled_from(AZURE_MISTRAL_MODELS), + ) + + +_AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL: Final[AzureDocumentIntelligenceModel] = ( + "azure_ai/doc-intelligence/prebuilt-layout" +) + + +def _document_intelligence_input( + model: AzureDocumentIntelligenceModel, + document: OcrDocument, + optional_params: Mapping[str, object] | None = None, +) -> AzureDocumentIntelligenceOcrSdkInput: + return AzureDocumentIntelligenceOcrSdkInput.model_validate( + {"model": model, "document": document, **(optional_params or {})} + ) + + +def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocumentIntelligenceOcrSdkInput]: + document: Final = pdf_document() + pages: Final = st.one_of( + st.sampled_from(((0,), (2, 0, 0, 1))).map(list), + st.just(["1", "2-4"]), + st.just("1-4, 5"), + ).map(lambda value: {"pages": value}) + features: Final = st.one_of( + st.sampled_from( + ( + ("languages",), + ("ocrHighResolution",), + ("barcodes",), + ("formulas",), + ("styleFont",), + ("keyValuePairs",), + ) + ).map(list), + st.just("languages, styleFont"), + ).map(lambda value: {"features": value}) + combined_query: Final = st.just({"pages": (0, 1), "features": ("languages", "styleFont")}) + return st.one_of( + st.sampled_from(AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS).map( + lambda model: _document_intelligence_input(model, document) + ), + st.just( + _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, + image_document("invoice 123", 24), + ) + ), + pages.map( + lambda optional_params: _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params + ) + ), + features.map( + lambda optional_params: _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params + ) + ), + combined_query.map( + lambda optional_params: _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params + ) + ), + st.just( + _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, + document, + {"req_format": "litellm"}, + ) + ), + ) + + +def azure_mistral_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("AZURE_AI_API_KEY") + base_url: Final = environ.get("AZURE_AI_API_BASE") + if not api_key or not base_url: + return () + return ( + OcrRecordingTarget( + name="azure-mistral", + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + azure_mistral_input_strategy(inline_image_data_uri), + ), + invocation=invoke_with_api_key(client, api_key), + required_inputs=AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, + ), + ) + + +def azure_document_intelligence_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_API_KEY") + base_url: Final = environ.get("AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT") + if not api_key or not base_url: + return () + return ( + OcrRecordingTarget( + name="azure-document-intelligence", + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), + strategy=cast(SearchStrategy[OcrSdkInputBase], azure_document_intelligence_input_strategy()), + invocation=invoke_with_api_key(client, api_key), + required_inputs=AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, + ), + ) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/base.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/base.py new file mode 100644 index 00000000000..1235ac1b5a2 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/base.py @@ -0,0 +1,48 @@ +from __future__ import annotations + +from typing import Annotated, Literal + +from pydantic import Field + +from ......shared.parity.fixture_models import ( + FixtureModel, + JsonSchemaDefinition, + JsonSchemaResponseFormat, + SdkInputBase, +) + +__all__ = ( + "DocumentUrlDocument", + "ImageUrlDocument", + "ImageUrlValue", + "JsonSchemaDefinition", + "JsonSchemaResponseFormat", + "OcrDocument", + "OcrSdkInputBase", +) + + +class OcrSdkInputBase(SdkInputBase): + fixture_only_fields = ("contract",) + + +class ImageUrlValue(FixtureModel): + url: str + detail: Literal["low", "auto", "high"] | None = None + + +class ImageUrlDocument(FixtureModel): + type: Literal["image_url"] + image_url: str | ImageUrlValue + + +class DocumentUrlDocument(FixtureModel): + type: Literal["document_url"] + document_url: str + document_name: str | None = None + + +OcrDocument = Annotated[ + ImageUrlDocument | DocumentUrlDocument, + Field(discriminator="type"), +] diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/common.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/common.py new file mode 100644 index 00000000000..b7040235868 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/common.py @@ -0,0 +1,101 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from functools import cache +from typing import Final, Literal, Protocol + +from hypothesis import strategies as st +from hypothesis.strategies import SearchStrategy + +from ......shared.parity.fixtures.pipeline import RecordingTarget +from ......shared.parity.fixtures.media import dummy_image_url, structured_pdf_data_uri +from .base import ( + DocumentUrlDocument, + ImageUrlDocument, + JsonSchemaDefinition, + JsonSchemaResponseFormat, + OcrSdkInputBase, +) + +OcrRecordingTarget = RecordingTarget[OcrSdkInputBase] + + +class OcrFixtureClient(Protocol): + def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: ... + + +class OcrSdkCall(Protocol): + def __call__(self, **kwargs: object) -> object: ... + + +@dataclass(frozen=True, slots=True) +class ApiKeyOcrInvocation: + client: OcrFixtureClient + api_key: str = field(repr=False) + + def execute(self, provider_url: str, case_input: OcrSdkInputBase) -> None: + self.client.execute(provider_url, self.api_key, case_input) + + +def image_document(text: str, font_size: int) -> ImageUrlDocument: + return ImageUrlDocument(type="image_url", image_url=dummy_image_url(text, font_size)) + + +def image_data_document(data_uri: str) -> ImageUrlDocument: + return ImageUrlDocument(type="image_url", image_url=data_uri) + + +@cache +def remote_pdf_document() -> DocumentUrlDocument: + return DocumentUrlDocument( + type="document_url", + document_url="https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", + ) + + +@cache +def pdf_document() -> DocumentUrlDocument: + return DocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri()) + + +def document_transport_strategy(inline_image_data_uri: str) -> SearchStrategy[ImageUrlDocument | DocumentUrlDocument]: + transports: Final[tuple[Literal["remote_image", "inline_image", "remote_pdf", "inline_pdf"], ...]] = ( + "remote_image", + "inline_image", + "remote_pdf", + "inline_pdf", + ) + + def as_document( + transport: Literal["remote_image", "inline_image", "remote_pdf", "inline_pdf"], + ) -> ImageUrlDocument | DocumentUrlDocument: + if transport == "remote_image": + return image_document("invoice 123", 24) + if transport == "inline_image": + return image_data_document(inline_image_data_uri) + if transport == "remote_pdf": + return remote_pdf_document() + return pdf_document() + + return st.sampled_from(transports).map(as_document) + + +def annotation_format(name: str) -> JsonSchemaResponseFormat: + return JsonSchemaResponseFormat( + type="json_schema", + json_schema=JsonSchemaDefinition( + name=name, + description="Extract the visible document fields", + schema={ + "type": "object", + "properties": {"title": {"type": "string"}}, + "required": ["title"], + "additionalProperties": False, + }, + strict=True, + ), + ) + + +def invoke_with_api_key(client: OcrFixtureClient, api_key: str) -> ApiKeyOcrInvocation: + return ApiKeyOcrInvocation(client=client, api_key=api_key) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/config.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/config.py new file mode 100644 index 00000000000..fe5cb566e36 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/config.py @@ -0,0 +1,13 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Final + +FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" +DEFAULT_FIXTURE_DIRECTORY: Final = Path(__file__).with_name("data") + + +def configured_fixture_directory() -> Path: + configured: Final = os.environ.get(FIXTURE_DIR_ENV) + return Path(configured).expanduser() if configured is not None else DEFAULT_FIXTURE_DIRECTORY diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/0b234402bdfd3be223e731051202f2996578fff02d93ec3011031ce2b4c511fc.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/0b234402bdfd3be223e731051202f2996578fff02d93ec3011031ce2b4c511fc.yaml new file mode 100644 index 00000000000..f7aa8598016 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/0b234402bdfd3be223e731051202f2996578fff02d93ec3011031ce2b4c511fc.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"image_limit":1}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d3637dc2c090-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:15 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-5a87-7148-8ae4-4e14967bebf3 + x-envoy-upstream-service-time: + - '226' + x-kong-proxy-latency: + - '19' + x-kong-request-id: + - 01a05e89-5a87-7148-8ae4-4e14967bebf3 + x-kong-upstream-latency: + - '227' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '56' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:15.394028+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + image_limit: 1 + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/1f3001924ef2c46d3eccdedd808c7ded7fea981215a478a12d8636eff8eb1157.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/1f3001924ef2c46d3eccdedd808c7ded7fea981215a478a12d8636eff8eb1157.yaml new file mode 100644 index 00000000000..f06a1033691 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/1f3001924ef2c46d3eccdedd808c7ded7fea981215a478a12d8636eff8eb1157.yaml @@ -0,0 +1,60 @@ +interactions: +- request: + body: '{"model":"invalid-ocr-model-for-parity","document":{"type":"document_url","document_url":"data:application/pdf;base64,JVBERi0xLjQKJZOMi54gUmVwb3J0TGFiIEdlbmVyYXRlZCBQREYgZG9jdW1lbnQgKG9wZW5zb3VyY2UpCjEgMCBvYmoKPDwKL0YxIDIgMCBSIC9GMiAzIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PAovQmFzZUZvbnQgL0hlbHZldGljYSAvRW5jb2RpbmcgL1dpbkFuc2lFbmNvZGluZyAvTmFtZSAvRjEgL1N1YnR5cGUgL1R5cGUxIC9UeXBlIC9Gb250Cj4+CmVuZG9iagozIDAgb2JqCjw8Ci9CYXNlRm9udCAvSGVsdmV0aWNhLUJvbGQgL0VuY29kaW5nIC9XaW5BbnNpRW5jb2RpbmcgL05hbWUgL0YyIC9TdWJ0eXBlIC9UeXBlMSAvVHlwZSAvRm9udAo+PgplbmRvYmoKNCAwIG9iago8PAovQ29udGVudHMgMTggMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXQo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKNSAwIG9iago8PAovQml0c1BlckNvbXBvbmVudCA4IC9Db2xvclNwYWNlIC9EZXZpY2VSR0IgL0ZpbHRlciBbIC9BU0NJSTg1RGVjb2RlIC9GbGF0ZURlY29kZSBdIC9IZWlnaHQgMzIwIC9MZW5ndGggNTYxNSAvU3VidHlwZSAvSW1hZ2UgCiAgL1R5cGUgL1hPYmplY3QgL1dpZHRoIDMyMAo+PgpzdHJlYW0KR2IiL2w2I3BIcSVSZE1bNDNaKzdadVpbLUFpNUtMMzIvY0tJbkxtaixYPmxxXEVxWFItTGs6PWw+I1lObGclcFhybTBwMGRhSFMvXHBpPC5cPyFyU1Y1PGpoNFMhcj5xc1hSVXAtN18zVjItWWksZChNMkNrWk5yLz9aTUFIV0FwJUthNz8nWS4zRzs6WW4zZlZ0TVtFXERkPkY3b2FlZjgrbkw6XERhWFhyMjpMN25AWUAzJ2ZWWktaMTh0aDtXMiVJTV5yWylmQk08RyhULVx0VCNERidTRWktKCM4YTApUiVEODMyazdNQGksOjU1aT5GZTouckk9amVcNE9rW19gLG0oMEI6MW1ZNXE4RURtVWRdazs/KCVIbS1zcFAsMDhuQzlbWy5yST1qZ1U2ZTIsX2JeUz1eNXNhXEwzXGtqQ1RmZkMhWWRyKXBVVj1iSk9janByOG9FJ2ZWWTBETy1KKlg3MTZwTV5yWykyb0kuMDJjL0pGSDU5VDtFLExuNzlBRVk1J2c3SSpERSdUUVhqSSpvI0pUSHNtb2BuOF9tcFlHVS5ddF1sQ2toOGNmMGckYzsiLWBHa0ZpUSppIjUyaWVlXSMlQD5baXJCMzg6bzpFbzkyazY/XFk5IUNiRWhwdURNXnJbKTJiQD07bzY6KD5EMG0lImdpLi8mXVFXWjFoWFBwYy8/Wk1BZ2YvUE5fVl4+PzJYRlJPRVBvPz9ubGgtaVglMlIuWmFmIkxFcmtpOFNBLl0lRFVsKSdpZURNL25saSEscmFZXWU9XjVzYVxNLzpcZzBhJSVTK0o4N1xEXlxnXVUlS1otVU5UW2A+KD4lJF5UQ2JyJih0I0QiZklKP25QYmw5Q3BiXyRDQ0I5WS4zSEcmcFooWD9cOlFOMzVcWT1RKzI/TTM5U3Q/PjpJY1M9NWo6KW09WVQ0cC8uMmxZL0NNMW8lRiJvL0ViJlY+JDteb1dpO2omWCM1PjxjOC9tPlhjXVs9LGExPkRpOkRnZk8nRFYtcWA/UXI9O1pQXT5fcmpIW1pdNWxyJTBJLm42Q2VYaS0oPTlNWiFFJnJRQWdFayI4a1lmQ1Iyb2M+LlMlUTcrazlZL0NLX1NLQ3NibzMvLS5HX1A+KlErLWdzO2RNZT80bHU9Tj9aPzxsNG5IZV5ZSHN0M3BGR1o+WSZjQT9yVWd1V209KiJwYms5O1hyKj8+J2FvLilZUihHMjdub3BrOWdBO1YvMFIlWExqbVJBOC0yL3FYQC4pIztiMjopPE9uVUJvQC4pbjxha3IwdSFWXCw0QC4qYTxha3NsaG9Cbk9wWUlNIXFQQmlBLGlLXms6PWtyL3BfYiQnLDhdPFVQPWtxPFhvLDNiLFdWKVNPLzhlWzBhRTVPMjVmbl5DTXFhR0JxY3FmI3JeS245Yj9UbDNpIiNPbWNMMTNdViFvbnM4bURAP2BvZWdwUSt1TGlwUmBzVSFTZ2UmJGhOKGhyWjdCX0knMUJrRTVVY2hiXEoka0haQjpcaz1TdFoyYWNIPV1RVTNmZ0RBN1xZL0NLX1BOND0+Q1kvRXBHOSZqMDhtRD9UOllydCRHLWVxJlFkVGlVYCEoKllqWG5cbFFpIkNAMjtSQ2U+UT9dZWhTUXBLXlY8RFhlN3UvKSJgT001KD9XIjpycWBndGNXbDs3aSp0YiFiXEwvVEZeQSs0XElvXm9LdW1QZGYqOyc6XFp1XWQscWFfK1BkYD9NYzE5SnRJU1ddNjRCJDc5bmVVQXFFVzAuYm10cClzVy5eKHFba1AhW29baXFmOkZpLC8tLz5NSVFiVkxrUSw3PllRI0krTD5sSkAlNHBrNTY8NVBjTmxaLktRMCdDOFg+JCFCNW8mMDMyRWpdUUtsMmZCcT8wKjwsPyskXycmJSY/Y0JTUjJDIz4jbEwjTER1UEY5cloqL0taP1dqO3BQM1lOUVRJU20rQlhnNmJTaWNVUidjUV8mbTsyVT1MVHItZDpfbmRhXiZoYV1McD9OOW9rUihTV21ITjNwTzlwMF9ia048Xy5hKUxfQ28sQk1YaEJkWFIncVhKYCQhNUhmbidORCs9OGUrVkdgNS09Z3NtcGU0bXJURCxrJEsmK1VaUWJXLl9kJj9XWkZINl4tU1pvW2k3KUljL3RrU3BRVTc0Ri10dGY3czlScmk8VFQpai5sdEgvSkM0Jz5HWFA/RVg7XGo9YTxjcFxybVZuIypkXl02IkdPUVJENWxJcjNkMl9xckk/bTlma2BqUnI6YEs+KnVWM3BvSy9uczpgLWRwSiM2cGM9SS5cOWpkaT9fNG9aSWs9WFldMCVTJURSVjNJSDFFVGBDTjBcZ2pBPzZYVWhCTUhQQllQYSRXP2E4TyVRcDslRWkrTV1HcGlSS1lnZjg8NUxmZTxyVltOaD1LNllhL20kRy0vPVRTRDwqP0doSkljMEN0UDInPDVLKixwW05CUGsiUzouaj88JWIlUyFWLTdqWnE9R29wQ25AR1lmVzhfdWdhRDBQTCdIO3FgalBrWF51XSZhKmc4Yi9KTUNLb2lOZSJbT25YTHNUSWZJQU5ZOXAuSFI/WCJWaEwpdVYhX3JaO29icyREXUJUZyMqcm4/P1tHNFZZY1JqJTdcYjtSSiM3JEhbQG9bZCtKWkJbcCtnYSUuZT9iJk8xMFVhOCpYTTxtPWI8P3U0RTpgLj1JLj9kP0dWaSJLRS4/K2tTNHUqcWlNT2NwcSJkKjlTLEE5QG9vXkNlWyJDanBnaF82WGJOSi1DakRXQ2BHXWtSXGRrYUpNYTYlWW4yWWU8ZCxtdGNKUjxzXD1jSTksQWhwQ2dQSmRsU2dTcmwxVz9fNHBgcmBWKzM5NnBlJCtGZVwtIXJUSCQkS0hLUUI6JEE0bGdfT2UnRUdMZlEpI2JrPHBQK0tZKyYvRVdrQEFNZVR0TkJsZk0nUzBeN0xyckpUYF5pPSM1U3FtdT1ES0Q7JFIxSHRaKTkhJT1SJUBTT3FoSHFjUkRhO1hWbmllK0ZCOl9pczY5WWpdKVkzNmRiTjQuNyI0Jyohci5XQUksOFk8WGU9SmxPX1ZEP21qaUxgakRpKT5WNGFaN1ReRyVLOWhIR0YhJDdHTjVQWlJrJGNtM25bckFFcHRAbitlOW8/MUpwUk51ZVZJZ3I+cUdnRlshYj9scmwsXT0/RyVBZSNbY2ksYEQ/U0lvb0RdYUg8VTJYTWQ5O2UmaT9vMUkzPHJROjpfR05KIlNsPlc/ZDNPIkxxLm4oWkxcLzYhNk1WIVlQTjZURVlAVDIyaFhmVVdqPV8/RVAuLkAoUURYLltjZUdAXF5ML2xpaFI8RGVObCEvLFQ6QUZOSyRiaWg0OFdlIiQ/LFpkR1M5Zls3WyluTW1dVCYwXVt1Y04pPDxGZjlwMyEjXkQ8aj03YUU2InEkVFpVakxyRTtNYzlCZG1gQ2QwNEYobDhmLixWXkNaImxxXCw+QDBIXWMndEkxP15OKE9YIWdRUkQ2LXJZaydDaSlrOUBwRkNgPW5tOkxfam85UFdFLV06by50O2o4aFdWLlNyXVIxZ0gsOTUqSXNMVGtsRUI9STRsZGw2Xz1vSTBIMUtoWFk9OEpANV5EcFBtb1RNWmpMczhTTXUzTm47cm4yP1lQZE0oSGI3JC5uVHVzcWpUWFBGcjhsJF5eZ0IyIzRdREUsXz1vSTBIMUs4TWY2MldTLFh1LDByUThLPV1PVWxJcCMhWHFQOEFWXW5zOmBtLF8nW21IT3U7MEsndWU/byZMaFJZUEFrOmI2dSRrcTkrdEdUImpEYShiQD84SWNXNFloTkBma0pcbWM5SUBtJ0NTa10zRW5DUClfXixrX1BpJnBVSW5vb3UrW10mXEddUlFkP1JISVguYWExVmYxITsiYW9lQF1DbHVaRHFmKGVSLlA0c21tNkZDLk9tLUU3Im4kJG9aN0luK1xWPDsyS3MnRDtrJlBcRjRZZU5qJUFkPVs7P28xSTNXVyRlVlMhbVheSyRiaWg4LEJoTERkSWhlJFYkP0BTYGZIMzRXYm5dMl4hVWRCVGckR3I6Uy8uXXRhdXU/U0lvb21qKkJHOSJvPi4kSGxkMVJPJT49SiI4QG9pNS0xb15ML2xpaEFRPWgyPmtfPC1uS2thQlRnIjFyKjlzKU9VRnVmcmRyTkNII05HJDxQPy9XZmE+bzAwVWE5VWVjJ0dXcFAsMDtZbCJbIjhiMWEobiIrRmNcVG5QKj9TSW9vbiJwIyRNdFc6JlVwMGIjVUlrJTVrak9AOypjP19MbmVbNjNhYTlTbkxONURjaTxtYjgwVWE5VT8yZ0YyamxhVzUvOllLTFAuVlYsKV48Y0UyYypyWSRWJD9AXXVBa1dRIz5RUEQjKTg8cEJsQ2NoWnJdP3FoSkRUbyZUMyxEaG1OJm8vIkhwMFVhOCpGKzddXTRtRGUzLGs7YjEtSFkjcWhmKmlPSFRqSE5xdXJpZW8hXDNNNmYuRitbbmU8Il8uXSJFNG9KKjwybCxXTVpjUVEuUk51Zj1KK01gXGw/VkVDbmVbNjNha1AodDwtXFJtPzlQNGEiZiJaW1tkbEBdaFVuRWQ6K0xoMUJUZyNWcXV0LlxoUiE3K25lWzYzYWtPYUBONF4jazQmN0pZXy5dIkUqVzFXNVMoK2UtcjoxWV8tSFRLWzAyZ1o+U2owKjZeTC9saWhIR0VGbnR1LUk8I0V1YXI4bCUpa1YkOSFtJS0/SSIxamM0NEYrIiovV2dqckYmZkM7QkFmMWpQSj9vTDxdSy9qVCQlazdeSD9mM3JtbSlnVldaWmxucz0pUz84bWZINlxBSEtkX1cmI3JFY19vaylvQ2I8VFA2LlhPY15TPlc4LmJobEEiWzhbbi4pYW4qQGNqUz86WUgqRl9pSS5nJ1tAPG5vInJkY0U8JGgxV2RdTiUrSSlqJWZzSCpkSGQtI1s1PGE8J0NxOllvJyM0XHBSTzRpOyR0bC5QLTljbGRWcFY8U3FxWWwqVilTKWdiTz9FMFQqJEhsZUpdR15dcDdeVXNHNj5qNktuczpfQlxBLT5KbSwxX2EyY1g7XmJFRm05P2lPNGRuWWhQPVZtLSpcYVcnSTRkUW1OclcwIUxFWWwqVikpcy1QUi4oOF1GT0lNckpyOGwkPmlwWGt1RzRWWW8pbTZGQ2ppTWw7bikhZ3NvVmRpalgwRE5gYWE8Nz9wPkdXOVcvcXNzWWwqVilIZkZNOWFMMV0+TztrJSNyOGwmNG1JMCM8Rm5WYnEpa3NTN2ppUC10bitpPi5wU2ExI2hRP187YWgtTU1hRXA9KU5zYEZRJUVpK01dVzskJDRlbDxBOydHXDBCREtgN0dLSl9qSCMqJC5fVksqNnJwby5DcVxwciVEO0wyXEhfJj5lVCJJZ0pBKidpO0hQXm4kSDI1WElmQWdlWl1YPCRnajAmKS9Jc0xVNm9KXS5vZzpUQGJRWW8rZ28mUDVmRGolY0BHcVw6VW1tQCRFakYtJyxUIk1JVm4pWlRLNy9sYzxyUUYodEgjKiJYYzFUa01yOGwmdGtcbW9CWzxaXVJEVldqb0gyI1oyR3JQWjc0bXJCUnI7REV1YWgtaixKQG1cSm8oUDs5LGskbzIxXC1SYmIyNS1rUC5aNm5hNWlmMFErcWdCOGIrcUBTV249OD04a0NYOGIrTTRyMmI+MmsqXWosbz0nK1NablwlXFNyTGtMbnRGSyRtZF1vSE1Dc0goXGBNa0lOU2ZCT3EpPFdoLk5dcidXYS5jNj4sU0VLS1g+Tj1vYC9hbzAzcDNKZE9lUSFkJTNsKkdjZW1TMTM3NyJcdUtENHF0WDU2R3FqVGAvXEgqISluVD1yJGNqamRwVEQpT3EmP2gsOklDPzpNWWtPSjstPGZlRDxGRCxBK0dOU1J1Oj9NcmU+N2hrRSVMak1ZZTIrWEtfVWNoanU+S1QuPW5pPmJvNy1wTWAySVgyQTIhJGZmKTxRRzRnJUluYFoiOG1FMkxVVGpbUGhGZVs3QjxLMSxFPC1FaWhuP15MMjZIIjU+U29EKEBKUk8/Ml9Mcl0+X3JqS1ZDRi1IPTgxUW8xbjYoOUBHRXUham1VWFNeNS9BJSxxYi5HVXAwZGltdW84ODQnMUZZXC8+WHM/ZnNVKF5ba11lJGZjc2wvOlRoW11fOksvLHFiLHFYMEROQCddKz5CYC5gMTUncGNWJVdpcWQ/RUZJWVdrSzddM0dKTHNAYEdOZjNxV0JuVG4qYzR1J1FIM1VuLDZNMUhNXWVkRiRhbDlFdDlXbXJlPU0uV2I2PktGJjFwbXJXXFxBWVwuc1ppLDI+XXIhZlZbcWdRTCljZVknUnEwUTNkbSFjbm9jZVhzUnE3Qm9RcmRNZyxjWCEpL3E3QkwtaVZlV3FUIj9oPW5lV0VlQEYqcmlhUyFrKVxcQXUsJFtXWmA7UWInclJKaGldPk0tOWxwTXJGZzhhQSQtaFNQbCddcGVoPDhpY2hSYkY8XmppZjwxXydhRWovM2UtXGpnO0xONyhXUStyJSppV1dyZyhXQDZnJSdIKHE0NnQ1PVMqVCRWUWpKPTVxKixoJVUoZj5UODFePl9yaktbU2xsLlxHVnVaQmtfXXJLVGYoSD1fbU4kUlYvWSk9c1UvcDwwS0V0QC5LMnVfXyVNSm0oREVcSXRxVSYucmYiMF9haCFXVVx0cG5BP05rKW1EOm0wYk5qRjchO0ZccT01cSosPD0xLDhGWyJPPkhQVF08LF04SVUpMHUxUWI+SSohXER0LEVLdTxNLiwqX2dRWS4uJi5ZZW1ZalNWUVtcWmFmIUFFS0xTNWs0S2guOF5IQ1JTJkMiMCowa2E0XmpVTUU9NTUmUjdjQkhBWCUuJFdaYWYhQUU8LUU9a004Ky5cIjpQUjJpNCQrbjZ0MWVqIy0xKz5dR3Q7Vz0jOVloKityZVQjREYnTTQ6RjRHMkFFLVtJJSslRUtkbigkTGQoPjRBPXEwYD4oPWdTIl09QGhYYjU2bW9gbjhQSVRRWVlOPzlTNWxbRnAnZlVzZ1g3MCRELi5lNjtNXnJaWDJjL0lySXJGLGpaYWYhQUVGW2w1QENZV0woMEI5OkRXJTk8IS1FLyppZWo1RE5WWSVfJ2V1QG9FUShaMypyPlEkUmolNnM0YzpoLicpMD5NMnIjWyhuKk83X0VLZG9lWTVRPmFQUD1ANmA+KD1nXTRdbkQyY1xfaEhQVF08LGpwRDJeVWAlZyVkNyYjRUtkbzVwQU5uTmpJT10pXERlaVRRUTFEN0BjU1FZWmFmIUFFVyU2YF9oSDJvV0ZwNGk6I1wzXnI4bzNiKlMuUixpWy9ZLjxvVzUzZjxgIUgtPzI4XU0zaUVzY2YwaE9gOWVvP2U9T1ZvMnNMIT5hWkkiRkVRKForSUg2c1RaYWYhQW0zX0NhbW9gbjhQUC9vcHJ0Iy1rPW0jfj5lbmRzdHJlYW0KZW5kb2JqCjYgMCBvYmoKPDwKL0JpdHNQZXJDb21wb25lbnQgOCAvQ29sb3JTcGFjZSAvRGV2aWNlUkdCIC9GaWx0ZXIgWyAvQVNDSUk4NURlY29kZSAvRmxhdGVEZWNvZGUgXSAvSGVpZ2h0IDMyMCAvTGVuZ3RoIDYzMTEgL1N1YnR5cGUgL0ltYWdlIAogIC9UeXBlIC9YT2JqZWN0IC9XaWR0aCAzNjAKPj4Kc3RyZWFtCkdiIi9sOTYpO2cmT2BoPmVYVDgrZFFNWkZQbm1RMU04ai8kJDM7ODVOYzljbTkjSVk2JWc2MnBGJS5ybT9YTmoycWQyU1I5MXFvTmxnT01pJyNVYStVUiUyXWQnPzQmOkhlYVJEZDdfWi1eNWdYXERlUXFoJiRJbEJcWzY/YEdOTktQPUQnJ10oVDkzRVNHRy5VXzBEX1FAME5NZ2JyRVwvJ3JaZG9dcmluaWpSZkxEIixdbzwoRjhZTV5yWmFZMGQ+OVxTOj9tZTdXZD9EOFdiI2kibj1CKkd0WChZLi8+WTk2XkUkUyJQY19JMjVvPipvKWA2UzNxMmRTJ3AnZy8/WkwiXDlqU0AqQi1xPD04LGomRVNHSEFZUEFFQFMzJCoqWmFmIydpTFg8RGNlS1hjPmhXaDFgR05MVV1bbXElXS9qaXJaYWYjJ2lMXHE/LE0tITtmQ2A5T0VTR0c8bV1yRExXL2VMZC8/WkwiXDZjWGhNPGJQX1U8aGhWZ09uaGhZMHUtLXA0am1eNSlVcS8pbTkocWk3USt0XidGUiJNXnJaYVkuSE5eL1Q3MEoiTUUoP01WRWspK2NVLWQpaSs7Vm1eJ0ZfW2MnW21qIVQwTWoqTV90bUJhPV5bYyw0dGojUWhRaFkmWiMvP1pMIlw9cT5xNE03cig4XGROSi5ySDtEWm5yZ09EbGRna1hqPyFhZ09nSTsoQSZYJGpmQGBwYHAnaD5palJnQzhgT2dgTDQvNS1JMjVvPjJWXnU/NFBucEttTjguS2llbEtGUmswT0heYDFVVzNuVl9aPl1KVDFhaT0rNE5xZDRFM19LNiJZLi8+WS0uNyNeXiUoMV9cIillbldUSGguYG5cSC4ncUguNSpHZmRDRElbU09idD8nOFVDTzw3JyI5QU1zIjNISGE6KFdeUl5qJz8wYzArKU9sVSlALnJjTUwmTURKV2E+cD9tZUV1RWtnckEuP2xnbCNUJTlLJTRwS1dSMFIjPkk1P11kLnJaakElamRhSCE7XGhEcjZYakdNJS9EWk90OStCOnNxUmpXNDhDYlVUbmwmYjw/PypxQVFLOmhLPFNmLCVeJCc3KDFaK0Y/Uk9gQHNGLGNOPGZjMFc7IU02TClpanMzdWY3RF1yOT4vMiY/WkYmLWs6VSRDSCI0NmxqbFwuc1xGLy8mcTNPJUs+Pz5qWmllbSdJaDNgUzhqMDQqJms6PF5GbWJHTERRVCYxJ1F1S2UhTVgvUkRTcCNDc1onZlVwKWsnaTU2TE5lbzRhQTlIWCNQTUM9NTtDZyspY0VMWS4iW2IoMEI5dWpZR0taN1dhRSRxXmVuWzVLZGk8amMvLEYoMEI7W0NATSxsRiw7aWcpTGZZMU07K0VQaipmYSlQckkiQ05eJFYpMldqai5VRzIuWFJsOmhIJW1GKFMyV2hTRFVHMi5YVGY1YCwhLXRgY21FMFIhWW4qI2xcXk1NUEI3VydFaVxvcj04WVwhLkYwLkNwOic+Tz1gLDNeLVB0IzM9a0BTWEtTLiIlVU07K0M6L1dJV1tjZk8iYzRXVUszJyI5U1NnKS41Q1REUT0sRkErWjgtI1I0MU5caCs3NSRiZzA0PWxUKSciOVNTOWNBXzJUMTdcQSojWlV1TTsrRGUuPzIzVzpHNldLQXAoSyYua1YpZGlAUUhRaFBabydUJ0g+dEZhZFk6YklMVXQtWkFicVJRQipoJkpfY29mXTNNOUpsMmooNGVncmVOZ1tObDspbWshXE9UV1NvJyk7cmlpO29xUE9qWzRcT281VTVrWDtsXDxoNlotYFhXJCV1Myo4bEBXdShPL1RRMmRPPUpSNmZYMChvWEs4Kiw5QUpfKW1lNl41PjlwWXEjM0loXDtsN2lOUFtrREFEWykqcXJkZ29vZFwvO0ZOU0FlUE47WkVidVtFTUhVUGgxOHUjKmU0LjdHYDJgRmtbamEiaSQwJEJjLGIuRU5YPmZKdXJadFBkUTZ1MDpgXk5jXFZfbD1iYVhpYVgtSV09P2ZIPkZgP2FgSCxzRFlYakRtT0leTzouKDYrclAxISw3RmB1NGBCPD1WOGIuPm1RNEtidV5Ga3U8Ij5pXiwuQ1hbVTxRSz09NEBjLzxucDVTVjlqVmk+UGU9OFYtSFlUS3FhUVhaaV1cRT1JVEtpRzFlRCJERF8uamkyUXNXOjosdFYoalJyMiM+b2cucGFnbGA0RUpQc2BsZz40VzNhaFItRV00SjNoS1ZKUWpQQj1GbjorXU4jQWReQVk0NGQ9TkBDO0FuIilJbXM4LnM/QC5QIk9tNy9VRDlbUCwyXCVPTm85OkRwSVw0X1VVbzo7VUludCgpOXBmRUhNImUxajldNlxZa2BZOy4xcFdzbEpJVi9FZ2ZYcz8lTkRsJCJWSikuKjAmQEhFZl1RJkhqc0IhU3JLXGpNOVtQLDJddD8pdGFVXUpIYy0qb1lQSyoqRT02LlVvUE42bFlgcGVjQEkyNF9aRHNtN0o8bU1SRURhJ1U4WUhLbT41PG8mTWZZITNTTFhJI2ciKjM2MjFlRCJ0RFtcbCQ+LU46YUVtTFI6NEJePUxha1REQ3BxJkddQ1xYbFFsOU5QSVAzWDxPaU5aa1NQckhqckhOUTIoaSh0Ry1IMkRTZU46SD0wIVY9R1EjaShVdSloLl9HZ0RdZkhXLW5JcURiU1AoSCkmJi1yLltPXF8iS2RfZ2NJMC05W1AsMic+LSd0T3QoVWlwPVssJ0g3aiwwZDNPIVldSjk+VSZwOyJgZnMuRVZHISYwR11FbTpfaFk9XSYwNmw4NzJzQFNMS2YzKWpSJWQxQ0FwUnReRDk7J25tQl1HU1tvX0ZSVS5PbyZIWGxpKVJIZj1pbzZEVz08bzA0PDhiMiJqQSwpVS9sQEE4Q0ZBJTY3Q2pobkw4Yi8wbGo3My0zT1cqbGZVbydeVGlsVzM9MWVIT2pEajcxJlthM10+T2JVTjJMKmsjSGxdJDo3XyJOJldONyplSGRhMDdiYklvVnVgMk5qSWNgOHA/Zy01VzxeJmlSaU1DLUgnaS9KbStMUHNYRVY2W1InaEZMLCJqOT4obiYvOVxNbUJhS2wpQzk7SVVJayU5R1tuZzhWby81XT1LPSQ6MmoyUTZIKGhsKmA7KzlOW2ExOG5rYCEwOk80Ty5HclZfYGhCYDkoIiguKGY9JjZkKFIyKEFtYCJEPDotS0RaRmptJlApKTVWPl8qX1dfMGkmX0lvQUY0bGZEcWcpWy5uJj8lPlYxJTNZNU9FdT4/OUErZzUhLEsxbnEpM3VcOktocV89Zmw7XEBkcl8tSFYyN05TdHFlRzZRYWFUWkhbbjE3QktXKCNCZEYiNXQ6KWpiPEInXUoyaj1qTHJNT19rISVsaEIlZzJuWSsvaVlJbFNZUE45RXUpdDNMLFpwISYqXTsxQTEuclMsLyxeSFRiYGMrVEUqUSVPRzBOViJVSCIyXllhYTlkQUw6YiRGU2cxJmlUIXEiWFVYOj80MEQzPihBIVZORXAzQFM7ViElOzwxN0JLV14ic1VvRDdXclk1KTJiSjRnMkxRNz9jZkdnKUlyc0lHUVlXVDsvN0FTbGFkOVUuT3E0bVokMDg1Tzk4Y2NPJiQ9Y1JXcC1kM051VkdbbjgqR008PmRGK1EsZXFPR0kqYi5FM09ha09dUG4kYCkoRitjYFQ4XSpdcF1KMmo9alIpI2ZpR08tRkgoUW1oaVNKM21AdTtWVGdXVVZTJTFndWRkY1kuVFxHJy5fUiVkMUNwQEg6Q2NgY1YpbWRrTm4mXmtYZVpiLyJdUE42VEZOVCU8ZW1PP2slakdkXCw4YCRXLCkiLCUpKk9YWGAyajJRNkdvTSo/XEc5WlhQXSZoRDg0SmcobUhvbmxybVBhO19KX08sN1BeRFhEZ1FFODkrY1QrJloiQ0JCNmUhRS1IU3A/TkZAcFlNW1VkPXBISGI+O2AlJ1AqY0RJbjZxInFUTDNvUVNOQEM7QTA5bkpORT8qRyVeSD5YO21YOkJbVkopLipDdTRxQnInUXMjKShmMidHZVNxTiFSRmooNWlEQS03WUVnXWRKdURlX2pIaFMuclVZbj5DKzFnKkhsLHNUJ29TISljSlJiPzxWN1VVTVlaQUlLTXBubzkpJjBCVGckUGhLVGpgVCVFc0VTQGczcURiU1AoSDAoI283U0RnPTpJNCp1L3BVcmEtSFJjVl1TbFNHcCQwa3Vqa1ZHNEB1O1ZULnJUZiszWGU0TlohaU9qbzU5ZiowaCU7ZVByQ21eaDVIOCZZKlFeVDc/Y2V0Mmw6bEhWMUcuXFdTJGNaVExuXnMpalxSI2RLJVUvMDZxJmItSFJjVihmUD5saiwxTGhPdUZxKlhmcCw6PSc8NTNdT2Q8c29ldUNXRio9W143NlcyYWBtNydwP2QkME43bigxMnIqJT10TkpsbC5Dcyg8KiVEQ1pvZCRFTzYycGBncWpVRmxERipmJW1bP25OXDc/Y2V0Mmw6aytlLGMlX1dTJGNaVExuXnMpa1kjb1c8NHNGQzw3RG4xZUQjIWokVjcnbjMobW9rMXFHMkB1O1ZULnJXKFlxKVMtNWNfS1VYLnNZV14tSFJjViRvOycyMDc9MFdtcmxlbDtgJSdQSCIwVmdYLlJBUUhXJ0ZTJ01YYVdmYyw5Vi5dMy5eSUtNcG5vOSkmMEJUZyQ4aEplVXNdLCYoXEghWmhVUDtmQFBQLls9QUUwU0FTMCJrMGdnQSlmMGAmcUNQVVJeKWIpdCkuIWY7UEIoZCpEQT4sTi9DPylbUDUyaE08az9yUURtYz00THIvQHU7VlQ5QkQvJzkrQF4tR3A9MiRQKmEzLlJGYDxIKlYpWlRSTnNQQzJ0QVVXTGNYVVZBMkhtN3A+LjNGa0NCMzhHcD0yJFAuLm5aJERkZC8hSTVuLTlwMnVqMnAxUz9fRF5KN0EySG03WTIwL0A3ZXBaV05GWGs7aE5tI2M/SUphOD0vZVJEODRKZyhtSDRxOjFvZ0RCWTc+aCYpLDY6J1I1XilRLi11I1wzb2w5MjFlSFAhaFFVNlZLS2ElSzZmKEAnUE47LGklSyJsXk8/VFNBNmYoQCdQTjZUPSVEK0hpcVJqMVBLS11xQFNBVkpPXTdEPE5IYjI0O0RdX3JOJ0BcMUQsYm1zJT9MPGAzSFxeMjhCUV9COGFfNmpNcE5nPzxcY01MYjwjdE8hMTdCS1dWO0JrPUJHXj9wVj0haEMxN0JLV05TWlkqQF4oOEViYSJBbmhPKCQvbm06TD1fb2k1PEI6IV9mSDAoIzdSTyU+amhKZGBOIThELjViaWg1RG1IcmFvI0dGYFtoVm50XEJkIS4xLC02LHRIIjJeWWFhOF8jQC4tcmg/SDouWDdQXkRYMD45QGJWMUlGYlRdI0IxMTdCSUFcMlBDZSFUKiJRcHNUX28pdTQwTFlMM0M9XStBOFVQTTFgPiNNTEVMPiRZLidKb2InKyguXkMqOkkzMTw4aychRlJRckhYVCcuO29Ncl9FYmF1dDZOWWYmXFNVOysoXVFKMVBdRGROPEQ+aSJRVjFlRCIlRGFbNSNcXi85IUQzO0puTiRjWilqLEYxVStTWFBxOTkjKW5QMGtQIzYraT0pJUhLXStRK15KbzdQXkRYMDdsLSdNdWhldWVNPS1saEZMLCJqPVUqcVZET0QzXU48U01hVyY/OFAuXDNaODxzSSNUPTwiQk5aXSo1VDhkTkldUzU0YFFOaEsmOldmRGwrOnFQRi9TTDxbLS9MPT5Sb0F1P1x0cVwsW1FbTDcsXkshcDsoKW43NStibkNBKiZcIVJOdWdXMmVvMT9naDJ0MkRRZ10tNz9jZldvTD9BSk9TZUJFVGhVaDNWSikuKkMjMTZzIydjYVwubyUsVWBZOy4xMm9jaltkUW02LEhVW1IzaE8oJC9uZzhTYDdpS0hzTltYWUliOFhBOjhgIi0hNVkyJ2djVSRbSUlzR09dLUhSNGklRWhbLkAsPz87N3MybGw7YCUpJitFJipuIUxVPkJmNV9qWGEsNjlkaEMuUSklJSg8PUo5ZWJJPy46JDk5W1AsMmg3WzZlOlk8PT5cYGA3PjJsOjhPXVk8NyxPIVNXIWhWOkpAYTtcJjc4Uj1vKi8uRWtAPzNdaSszb2w4TFJPIyknMmNgRVwyRDVBXDpFQiNHJ01YYkJgPl1hYCNiLjA5XTVfVFZgWTsuMXBXc2w2Y2NgUDxXT1QhQS1aO10rLF5FMXVZcTZGR11QUUIiNmUocCxkM08iSF1sPjUqakxnLk5xYmJLVDtgJSkmNGA7PDQiJCYiXmY2RXRaYFk7LjFHSmJBVGBXND1YYk0+LWtIL2ZXNlAxU00vXCojKCNhcj8rTFAuXE5jQlRROyNIL2YkJVAxUykjNG49TC1HVD9RY0dvYjlYSCtnbDwtRWpxLzkmQTJmUE42bGFJNyRHKmY/SkBuUk8lPzdEWVtEN25FWmZvOi1iJChyUDEhLDhDXTs3YEI8PVY4YjBVVVFCLmQqWG9iNlk5SmwsZCslIk07PEJbY29RKWY8MTw7MEdkbjdwSyklPk5Ic1YkWnQxOD1uIz1cWCNIUzNvcSojXS5BJE8uSVJuPCxdQU1yYjRDTDs9Uz9CbipkUi1kUC5XKmhXNkhpcTksWjotLUVlIlVQLipndEttQ1hbWEsoNFttLjgtVU5QXCYkSCZpNkFCXjFbPFU5IiF0XUE9Yy0vcmkhUk5SMC4+a2hjJmlPQlhuR1s9KkswXVJiJT9tSShXMVFbLGEqZiYiMmYpSi1FS0tcWi5fT3VNLyg3aUZDNWBAXiciTmdbUT0+PldhPWE9RC1PW3JDIVwlRUNmITdXX3NWYklMVXQtWj03W29fPW06J2MyTy1VKDJJZFYyV2k5UD8yIUdEUik9YU5Vc0ZxPW51PFpnUmFuNiVYSSo9Ti8qKkA9NXBbT1ZQdXI+ZldNKDY4cyRScltsbFo/KlYuM1ZYaEtiMlI0RTozRXRkRHBhcjs7LUVAdCgvLSNTJ0lab1s5LWVfMyhzWygtZ1k0Wj89XG8jPVJFKnNDTSxNOytVQFZbZSFpPzcvNm4zRVAycUUkTmpVRF0lZkVxOGRWJ0RMOE91M29nYENTW1RWZVkvRzInb2k2ZFtFUywsPF9zdW5eaCJGdUNGaVlMTElDP1ddTVJhQS80bm5ZNyM5UzIvcHFEWyNBWmlzZ2NaTCtBR3BCSyQjXm1JaWd0K1MvRGc+aWZYKGdhZFA/MiFVbVwpXSZOUmBKKVQ1U2dYRUxGQWxkazVSS1puX0UpYD4oPWZdL1Q/UGBhSnR1YzwiN2pLPURrYTA1anRoRnNyP2U1KVVxb1YsbGJkaEZOX3RAQHRMUmYydFpeSl9WMilkTz1JZ1xEbDVjSU49ZmhPWD8oKTUpVXJaO0FoK2tYWitbZSZUa2JITUFzTEFXMiYsZS0tUC1cL001OEU0N1swZCteLyc4VjQ3JWhlLVtjWnFndUw3Qm9iXU5EYUFIOTZjQmQhSlFtQTc1IipcKWVPbms5NF1nWE9rRV1KSWY+bjdAS1tkXXU5JjAsRktNMXFuMS5nP3JGIVleTjAlPG4kOXB1WGJCQHUnXi0jUyZeSmldWHRRM1ZkSGdPPzxEQWArY0YuVnAhaT8hKjkjJyI6JDVuOiZAPFNWSCxhP1pGJG9pYDwkJywtVSwoTXNZbzInZlVxUVUoNnExRyVuMU9ZSU5RckVWJihQcVc5WydeViVuUE1WRilaWDpFOmBwNzNIIz1eNXM5RUFRLyhRJDZyWytGUCMsZ2gsPXVGaCRMPGgndTBsYD4saigyYUpKSUdySVg8ZW45IUFEOHFROWEnbWk8bS9jb2A+XUg2UTZlL0g0cForbVFaZDdXZ2dQZClMTmtOKVxZP3RdNy8/Wk0tXDNZJSoydEVSLyJNKWs8TVZGKC9Ub1JbOS9wZFRlWmFmIVFqMTs6JC5CYU8rXlgobnNnUGtJISVTKEVDNSlVcS81SGFHc3JGcmUtcUNKZVpjKyNCTWZ0bjJFV0goWShgPixqImZCWDE1cEBpSnJqMk85QC5ySTAyaVAjNislP2VgLUkyNW8+QiYmUlhyZi5wRk5JMU9SWS4yYGRnWGdobyVAK3JgSTI1bz5CJigpQ3JsUFstN1k8bV8uckkwMkwsc2QnJztYJEtjZjBpOk8xVE4uSXJdVitkQUc3TE1WRikqMm8lQDI5Y0JUIS8/Wk0tXDQuIVZlYnIuakhoOkhHRUxVciIyVFY0U15IUlErRERTRmEkZCRoMWZrNTxERDhvIy43YjloXi5ySS1xTlxQa1M9XjVzOUVMM0trVjU6I2JwJkZdYk0jUkpsM1NKSH4+ZW5kc3RyZWFtCmVuZG9iago3IDAgb2JqCjw8Ci9CaXRzUGVyQ29tcG9uZW50IDggL0NvbG9yU3BhY2UgL0RldmljZVJHQiAvRmlsdGVyIFsgL0FTQ0lJODVEZWNvZGUgL0ZsYXRlRGVjb2RlIF0gL0hlaWdodCAxMjAgL0xlbmd0aCAxMTI3IC9TdWJ0eXBlIC9JbWFnZSAKICAvVHlwZSAvWE9iamVjdCAvV2lkdGggMTIwCj4+CnN0cmVhbQpHYiIwUzk5LE4tJi1VQFxFIXAkZ3BWMjVzMGFrMXRuNyYxL2xDZkBjJmMmSVdZbGw5QUpWaz88RT8nSVdzI003Qj9BUzlPSD9qJUZDcWg3OmhvRkArSiVYZ2xMUk9Cay5uPTJNJXNITUo7bFFaVWdRNl4xLEZvb0BIMUJEVVM6dUdyIkIsaFBRS1ojJixIMyVOJGYwKz1ARlNlV2peTi89MSI4Ikw8RTs+KiIvbEdhQ1IkTW5nIW5pSyU2Z0QpV1JTJkBEKSsmXDRSMForQXM8KnMmcDcsY08yYlpeSz1baGFkbTktKVZmKFxhKEQsXzxXSSRXbUFzX1peVz5kam1WSDJlZjZnSGhPP2BpOExnVlRBYDxON1o7STIiLm1QPFNzIVguMiYpNDJbMypjSFwyMVNKZktpQjI+QTBjNlFxdVphajE0Qi0rQTlmaUc7TDFETGVlQ2FgSDJSPm0vR2VsRkJhKi5TSD5RNSZiMz9scSxnRFdyXTBPbD1UTFcuPzkoazhEY08mPF8qWUM6RkpvXE0jZ20sMkpZRDw1RDJrbXJLcj1fOU9kYmwiKEBZO1hLQFMpUHJsVlFNKEQ7XSo8MWYjQHUlXFR1Mi9wQ2wrPScxMy0xaTwwRl5UTEgxLGUkV0A2ayM+aVY0NU1LPkhGN1BGPW84XCNBOyttIVBGLCFVJE9WYCwqaTFRMUhZKCFpYGB1LjJxXVlpYVM6MjlVNGdZIzkmXG5YPV1XM3RmLEIqNF5gclNQbihDPjEpQVNbSHJPalIlbyZJTjNNbXFsKG1aREooPylFN2pSRVRwPzUkU14jWmFAQ05RUEgsJVtQSXI9NUg2QyckOkIqZSZRZTk0OyloY2pyMSdzZD4hbTxtcUYjO2Y3cUIsZEU4cjlcO2MxTC4+WUQ5MUZIKE8sUXFYNyEzdFpRRT1GXXJYIXM9XC02JzFEWCteXW9GbE49XGNZZDFDbVc4czF1dXNBNiRjT2s4N19pRmcqWjM0Y1hmdUYzYSplPzo+Jkw7TTJJRDlOUyxgYmwvVGQtXlFFS21NcGFeVGhoaDQoaWpyIW9fNW9NZEU9UEBXdTxrNTM/PitCMjtDM19XJFhDUTM/PiJLMjorQFNWKHNwTj1dTGpjczNdbDIjTkJ0JCdQRDo1O1UuI0hyY25yIVgjVURdNDZPJCUvKlAzU3A8NzspPG4hSS9ENF0/WFdON246Zyw/LkA5UCVpTFs6KkJrTlQvNGxDQC9acSFxaSxDZT5kZGtkNExsbmUxJl9RTV9mPjtQbDdWRTFNWldKbXInamxMcW1MUSo0X2lOaHItYmNKX1xjLldIIitwNkxPMSsxJjcnVzZaMjotUUZZNTksJ1M3PnE+QUhKLDdlYE8uKSJeSCtPPiNYPitLX0VPTjA+XWAqVHVfYUhOLlVEVnIhZWFNWE8lZUZDV09TMGNcVGE8UjdIZS9kXkhiVW5OMmZcJk9HSUdWb3QsMDhmODA8K3NSSzJtXkZ+PmVuZHN0cmVhbQplbmRvYmoKOCAwIG9iago8PAovQ29udGVudHMgMTkgMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXSAvWE9iamVjdCA8PAovRm9ybVhvYi4yYTg4YWM2NzEyYzljYTcyN2U0YWQ4ZjI1YmNiZGFlMiA3IDAgUiAvRm9ybVhvYi42YmU0MGVlYWFmMGQ4MWQxODc1MGRjMGE3ZjlkYzIxMyA1IDAgUiAvRm9ybVhvYi5lNWMzMTQ3ZDE4Zjk4NjdhNGYxMDhkM2E1NWEyNDhiNCA2IDAgUgo+Pgo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKOSAwIG9iago8PAovQSA8PAovUyAvVVJJIC9UeXBlIC9BY3Rpb24gL1VSSSAoaHR0cHM6Ly9leGFtcGxlLmNvbS9pbnZvaWNlcy9JTlYtMjA0OCkKPj4gL0JvcmRlciBbIDAgMCAwIF0gL1JlY3QgWyA0NSA1NzUgMzAwIDU5MCBdIC9TdWJ0eXBlIC9MaW5rIC9UeXBlIC9Bbm5vdAo+PgplbmRvYmoKMTAgMCBvYmoKPDwKL0MgWyAuODMgLjg5IC45NSBdIC9Db250ZW50cyAoVG90YWwgaGlnaGxpZ2h0ZWQgZm9yIHJldmlldykgL1F1YWRQb2ludHMgWyA0MCA1NzkgNTQwIDU3OSA0MCA1NTUgNTQwIDU1NSBdIC9SZWN0IFsgNDAgNTU1IDU0MCA1NzkgXSAvU3VidHlwZSAvSGlnaGxpZ2h0IC9UeXBlIC9Bbm5vdAo+PgplbmRvYmoKMTEgMCBvYmoKPDwKL0MgWyAwIDAgMCBdIC9Db250ZW50cyAoVmVyaWZ5IHRoZSBoaWdobGlnaHRlZCB0b3RhbCkgL1F1YWRQb2ludHMgWyA1MjAgNTI1IDU0MCA1MjUgNTIwIDU0NSA1NDAgNTQ1IF0gL1JlY3QgWyA1MjAgNTI1IDU0MCA1NDUgXSAvU3VidHlwZSAvVGV4dCAvVHlwZSAvQW5ub3QKPj4KZW5kb2JqCjEyIDAgb2JqCjw8Ci9Bbm5vdHMgWyA5IDAgUiAxMCAwIFIgMTEgMCBSIF0gL0NvbnRlbnRzIDIwIDAgUiAvTWVkaWFCb3ggWyAwIDAgNjEyIDc5MiBdIC9QYXJlbnQgMTcgMCBSIC9SZXNvdXJjZXMgPDwKL0ZvbnQgMSAwIFIgL1Byb2NTZXQgWyAvUERGIC9UZXh0IC9JbWFnZUIgL0ltYWdlQyAvSW1hZ2VJIF0KPj4gL1JvdGF0ZSAwIAogIC9UcmFucyA8PAoKPj4gL1R5cGUgL1BhZ2UKPj4KZW5kb2JqCjEzIDAgb2JqCjw8Ci9Db250ZW50cyAyMSAwIFIgL01lZGlhQm94IFsgMCAwIDYxMiA3OTIgXSAvUGFyZW50IDE3IDAgUiAvUmVzb3VyY2VzIDw8Ci9Gb250IDEgMCBSIC9Qcm9jU2V0IFsgL1BERiAvVGV4dCAvSW1hZ2VCIC9JbWFnZUMgL0ltYWdlSSBdCj4+IC9Sb3RhdGUgMCAvVHJhbnMgPDwKCj4+IAogIC9UeXBlIC9QYWdlCj4+CmVuZG9iagoxNCAwIG9iago8PAovQ29udGVudHMgMjIgMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXQo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKMTUgMCBvYmoKPDwKL1BhZ2VNb2RlIC9Vc2VOb25lIC9QYWdlcyAxNyAwIFIgL1R5cGUgL0NhdGFsb2cKPj4KZW5kb2JqCjE2IDAgb2JqCjw8Ci9BdXRob3IgKExpdGVMTE0gT0NSIGZpeHR1cmUgZ2VuZXJhdG9yKSAvQ3JlYXRpb25EYXRlIChEOjIwMDAwMTAxMDAwMDAwKzAwJzAwJykgL0NyZWF0b3IgKGFub255bW91cykgL0tleXdvcmRzIChPQ1IsIGludm9pY2UsIHRhYmxlLCBmaWd1cmUsIGFubm90YXRpb24pIC9Nb2REYXRlIChEOjIwMDAwMTAxMDAwMDAwKzAwJzAwJykgL1Byb2R1Y2VyIChSZXBvcnRMYWIgUERGIExpYnJhcnkgLSBcKG9wZW5zb3VyY2VcKSkgCiAgL1N1YmplY3QgKFNlbWFudGljIE9DUiBjb3ZlcmFnZSBmb3IgdGFibGVzLCBmaWd1cmVzLCBhbm5vdGF0aW9ucywgYW5kIG1ldGFkYXRhKSAvVGl0bGUgKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgL1RyYXBwZWQgL0ZhbHNlCj4+CmVuZG9iagoxNyAwIG9iago8PAovQ291bnQgNSAvS2lkcyBbIDQgMCBSIDggMCBSIDEyIDAgUiAxMyAwIFIgMTQgMCBSIF0gL1R5cGUgL1BhZ2VzCj4+CmVuZG9iagoxOCAwIG9iago8PAovTGVuZ3RoIDIyODAKPj4Kc3RyZWFtCjEgMCAwIDEgMCAwIGNtICBCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVAowIDAgMCByZwpCVCAvRjEgMTEgVGYgMTMuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc3MCBUbSAoUXVhcnRlcmx5IE9wZXJhdGlvbnMgUmVwb3J0KSBUaiBUKiBFVApCVCAvRjIgMTYgVGYgMTkuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc0NSBUbSAoSW52b2ljZSBTdW1tYXJ5IGFuZCBMaW5lIEl0ZW1zKSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgMSBvZiA1KSBUaiBUKiBFVApuIDQ1IDYyNSBtIDQ1IDczMCBsIFMKbiAyNDUgNjI1IG0gMjQ1IDczMCBsIFMKbiA0MDUgNjI1IG0gNDA1IDczMCBsIFMKbiA1NjUgNjI1IG0gNTY1IDczMCBsIFMKbiA0NSA3MzAgbSA1NjUgNzMwIGwgUwpuIDQ1IDY5NSBtIDU2NSA2OTUgbCBTCm4gNDUgNjYwIG0gNTY1IDY2MCBsIFMKbiA0NSA2MjUgbSA1NjUgNjI1IGwgUwpCVCAxIDAgMCAxIDU1IDcwNyBUbSAoSXRlbSkgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNzA3IFRtIChRdWFudGl0eSkgVGogVCogRVQKQlQgMSAwIDAgMSA0MTUgNzA3IFRtIChBbW91bnQpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNTUgNjcyIFRtIChEb2N1bWVudCBhbmFseXNpcykgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNjcyIFRtICgyKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA2NzIgVG0gKDEyMC4wMCkgVGogVCogRVQKQlQgMSAwIDAgMSA1NSA2MzcgVG0gKE9DUiB2ZXJpZmljYXRpb24pIFRqIFQqIEVUCkJUIDEgMCAwIDEgMjU1IDYzNyBUbSAoMSkgVGogVCogRVQKQlQgMSAwIDAgMSA0MTUgNjM3IFRtICg4MC4wMCkgVGogVCogRVQKbiA0NSA0OTUgbSA0NSA2MDAgbCBTCm4gMjQ1IDQ5NSBtIDI0NSA2MDAgbCBTCm4gNDA1IDQ5NSBtIDQwNSA2MDAgbCBTCm4gNTY1IDQ5NSBtIDU2NSA2MDAgbCBTCm4gNDUgNjAwIG0gNTY1IDYwMCBsIFMKbiA0NSA1NjUgbSA1NjUgNTY1IGwgUwpuIDQ1IDUzMCBtIDU2NSA1MzAgbCBTCm4gNDUgNDk1IG0gNTY1IDQ5NSBsIFMKQlQgMSAwIDAgMSA1NSA1NzcgVG0gKEl0ZW0gY29udGludWVkKSBUaiBUKiBFVApCVCAxIDAgMCAxIDI1NSA1NzcgVG0gKFF1YW50aXR5KSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA1NzcgVG0gKEFtb3VudCkgVGogVCogRVQKQlQgMSAwIDAgMSA1NSA1NDIgVG0gKEZpeHR1cmUgdmFsaWRhdGlvbikgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNTQyIFRtICgzKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA1NDIgVG0gKDQ1LjAwKSBUaiBUKiBFVApCVCAxIDAgMCAxIDU1IDUwNyBUbSAoUHJvdmlkZXIgcmV2aWV3KSBUaiBUKiBFVApCVCAxIDAgMCAxIDI1NSA1MDcgVG0gKDEpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDE1IDUwNyBUbSAoMjUuMDApIFRqIFQqIEVUCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiAxLjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiAxLjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiAxLjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiAxLjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiAxLjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiAxLjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiAxLjc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiAxLjg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKMTkgMCBvYmoKPDwKL0xlbmd0aCAxNzIzCj4+CnN0cmVhbQoxIDAgMCAxIDAgMCBjbSAgQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKMCAwIDAgcmcKQlQgL0YxIDExIFRmIDEzLjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NzAgVG0gKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgVGogVCogRVQKQlQgL0YyIDE2IFRmIDE5LjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NDUgVG0gKFJldmVudWUgQ2hhcnQgYW5kIEZvcm11bGEgUmV2aWV3KSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgMiBvZiA1KSBUaiBUKiBFVAouMiAuNDUwOTggLjg1MDk4IHJnCm4gNzAgNjEwIDY1IDcwIHJlIGYqCm4gMTcwIDYxMCA2NSAxMTUgcmUgZioKbiAyNzAgNjEwIDY1IDkwIHJlIGYqCm4gMzcwIDYxMCA2NSAxMzAgcmUgZioKMCAwIDAgcmcKQlQgMSAwIDAgMSA5MCA1OTAgVG0gKFExKSBUaiBUKiBFVApCVCAxIDAgMCAxIDE5MCA1OTAgVG0gKFEyKSBUaiBUKiBFVApCVCAxIDAgMCAxIDI5MCA1OTAgVG0gKFEzKSBUaiBUKiBFVApCVCAxIDAgMCAxIDM5MCA1OTAgVG0gKFE0KSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDU1MCBUbSAoRm9ybXVsYTogZ3Jvc3MgbWFyZ2luID0gXChyZXZlbnVlIC0gY29zdFwpIC8gcmV2ZW51ZSkgVGogVCogRVQKcQo3MCAwIDAgNzAgNDU1IDY1NSBjbQovRm9ybVhvYi42YmU0MGVlYWFmMGQ4MWQxODc1MGRjMGE3ZjlkYzIxMyBEbwpRCnEKNzAgMCAwIDYyIDQ1NSA1NjUgY20KL0Zvcm1Yb2IuZTVjMzE0N2QxOGY5ODY3YTRmMTA4ZDNhNTVhMjQ4YjQgRG8KUQpxCjQ1IDAgMCA0NSA0NTUgNTAwIGNtCi9Gb3JtWG9iLjJhODhhYzY3MTJjOWNhNzI3ZTRhZDhmMjViY2JkYWUyIERvClEKQlQgL0YxIDEwIFRmIDEyIFRMIEVUCkJUIDEgMCAwIDEgNDUgNDcyIFRtIChTZWN0aW9uIDIuMTogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNDQ0IFRtIChTZWN0aW9uIDIuMjogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNDE2IFRtIChTZWN0aW9uIDIuMzogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzg4IFRtIChTZWN0aW9uIDIuNDogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzYwIFRtIChTZWN0aW9uIDIuNTogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzMyIFRtIChTZWN0aW9uIDIuNjogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzA0IFRtIChTZWN0aW9uIDIuNzogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMjc2IFRtIChTZWN0aW9uIDIuODogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCiAKZW5kc3RyZWFtCmVuZG9iagoyMCAwIG9iago8PAovTGVuZ3RoIDI0NjYKPj4Kc3RyZWFtCjEgMCAwIDEgMCAwIGNtICBCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVAowIDAgMCByZwpCVCAvRjEgMTEgVGYgMTMuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc3MCBUbSAoUXVhcnRlcmx5IE9wZXJhdGlvbnMgUmVwb3J0KSBUaiBUKiBFVApCVCAvRjIgMTYgVGYgMTkuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc0NSBUbSAoS2V5IFZhbHVlcywgTGluaywgSGlnaGxpZ2h0LCBhbmQgQ29tbWVudCkgVGogVCogRVQKQlQgL0YxIDkgVGYgMTAuOCBUTCBFVApCVCAxIDAgMCAxIDQ1IDMwIFRtIChDb25maWRlbnRpYWwgfCBQYWdlIDMgb2YgNSkgVGogVCogRVQKQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKQlQgMSAwIDAgMSA0NSA3MDAgVG0gKEludm9pY2UgTnVtYmVyOiBJTlYtMjA0OCkgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA2NzUgVG0gKFB1cmNoYXNlIE9yZGVyOiBQTy00MDk2KSBUaiBUKiBFVAouOTQ5MDIgLjkwMTk2MSAuMzUyOTQxIHJnCm4gNDAgNTU1IDUwMCAyNCByZSBmKgowIDAgMCByZwpCVCAxIDAgMCAxIDQ1IDU2MCBUbSAoSGlnaGxpZ2h0ZWQgdG90YWwgcmVxdWlyaW5nIHJldmlldykgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA1MzAgVG0gKFJldmlld2VyIGNvbW1lbnQ6IHZlcmlmeSB0aGUgaGlnaGxpZ2h0ZWQgdG90YWwgYmVmb3JlIGFwcHJvdmFsKSBUaiBUKiBFVAoxIDAgMCByZwpCVCAxIDAgMCAxIDQ1IDQ5NSBUbSAoUmV2aXNlZCB0b3RhbDogMjQ1LjAwKSBUaiBUKiBFVApuIDQ1IDUwMSBtIDE1MCA1MDEgbCBTCjAgMCAwIHJnCkJUIDEgMCAwIDEgNDUgNTc1IFRtIChodHRwczovL2V4YW1wbGUuY29tL2ludm9pY2VzL0lOVi0yMDQ4KSBUaiBUKiBFVApxCjEgMCAwIDEgOTAgMTMwIGNtCm4gMTggMCAyLjQgNzAgcmUgZioKbiAyMS42IDAgMS4yIDcwIHJlIGYqCm4gMjUuMiAwIDMuNiA3MCByZSBmKgpuIDMxLjIgMCAzLjYgNzAgcmUgZioKbiAzOC40IDAgMi40IDcwIHJlIGYqCm4gNDIgMCAxLjIgNzAgcmUgZioKbiA0NC40IDAgMi40IDcwIHJlIGYqCm4gNDkuMiAwIDIuNCA3MCByZSBmKgpuIDUyLjggMCAyLjQgNzAgcmUgZioKbiA1Ny42IDAgMy42IDcwIHJlIGYqCm4gNjIuNCAwIDIuNCA3MCByZSBmKgpuIDY2IDAgMy42IDcwIHJlIGYqCm4gNzAuOCAwIDIuNCA3MCByZSBmKgpuIDc2LjggMCAxLjIgNzAgcmUgZioKbiA4MS42IDAgMS4yIDcwIHJlIGYqCm4gODQgMCAzLjYgNzAgcmUgZioKbiA4OC44IDAgMi40IDcwIHJlIGYqCm4gOTIuNCAwIDMuNiA3MCByZSBmKgpuIDk3LjIgMCAxLjIgNzAgcmUgZioKbiA5OS42IDAgMy42IDcwIHJlIGYqCm4gMTA0LjQgMCAyLjQgNzAgcmUgZioKbiAxMTAuNCAwIDEuMiA3MCByZSBmKgpuIDExMi44IDAgNC44IDcwIHJlIGYqCm4gMTE4LjggMCAzLjYgNzAgcmUgZioKbiAxMjMuNiAwIDMuNiA3MCByZSBmKgpuIDEyOC40IDAgMi40IDcwIHJlIGYqCm4gMTMyIDAgMy42IDcwIHJlIGYqCm4gMTM2LjggMCAyLjQgNzAgcmUgZioKbiAxNDEuNiAwIDEuMiA3MCByZSBmKgpuIDE0NS4yIDAgMy42IDcwIHJlIGYqCm4gMTUwIDAgMi40IDcwIHJlIGYqCm4gMTU2IDAgMy42IDcwIHJlIGYqCm4gMTYwLjggMCAxLjIgNzAgcmUgZioKbiAxNjMuMiAwIDIuNCA3MCByZSBmKgpRCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiAzLjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiAzLjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiAzLjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiAzLjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiAzLjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiAzLjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiAzLjc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiAzLjg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKMjEgMCBvYmoKPDwKL0xlbmd0aCAxNDk0Cj4+CnN0cmVhbQoxIDAgMCAxIDAgMCBjbSAgQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKMCAwIDAgcmcKQlQgL0YxIDExIFRmIDEzLjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NzAgVG0gKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgVGogVCogRVQKQlQgL0YyIDE2IFRmIDE5LjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NDUgVG0gKEFwcHJvdmFsIFNpZ25hdHVyZSBhbmQgV2F0ZXJtYXJrKSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgNCBvZiA1KSBUaiBUKiBFVApxCi44Mjc0NTEgLjgyNzQ1MSAuODI3NDUxIHJnCkJUIC9GMiA1NCBUZiA2NC44IFRMIEVUCi45MDYzMDggLjQyMjYxOCAtMC40MjI2MTggLjkwNjMwOCAxMTAgMzkwIGNtCkJUIDEgMCAwIDEgMCAwIFRtIChEUkFGVCkgVGogVCogRVQKUQowIDAgMCByZwpCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVApCVCAxIDAgMCAxIDQ1IDYzNSBUbSAoQXBwcm92ZWQgYnk6IEpvcmRhbiBMZWUpIFRqIFQqIEVUCm4gNDUgNjEwIG0gMzEwIDYxMCBsIFMKbiA1NSA1OTUgbSA3NSA2MjUgMTEyIDYwMiAxNTUgNjAwIGMgUwpCVCAxIDAgMCAxIDQ1IDU4MCBUbSAoU2lnbmF0dXJlKSBUaiBUKiBFVApCVCAvRjEgMTAgVGYgMTIgVEwgRVQKQlQgMSAwIDAgMSA0NSA0NzIgVG0gKFNlY3Rpb24gNC4xOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA0NDQgVG0gKFNlY3Rpb24gNC4yOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA0MTYgVG0gKFNlY3Rpb24gNC4zOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzODggVG0gKFNlY3Rpb24gNC40OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzNjAgVG0gKFNlY3Rpb24gNC41OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzMzIgVG0gKFNlY3Rpb24gNC42OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzMDQgVG0gKFNlY3Rpb24gNC43OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAyNzYgVG0gKFNlY3Rpb24gNC44OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKIAplbmRzdHJlYW0KZW5kb2JqCjIyIDAgb2JqCjw8Ci9MZW5ndGggMTMyNwo+PgpzdHJlYW0KMSAwIDAgMSAwIDAgY20gIEJUIC9GMSAxMiBUZiAxNC40IFRMIEVUCjAgMCAwIHJnCkJUIC9GMSAxMSBUZiAxMy4yIFRMIEVUCkJUIDEgMCAwIDEgNDUgNzcwIFRtIChRdWFydGVybHkgT3BlcmF0aW9ucyBSZXBvcnQpIFRqIFQqIEVUCkJUIC9GMiAxNiBUZiAxOS4yIFRMIEVUCkJUIDEgMCAwIDEgNDUgNzQ1IFRtIChBcHBlbmRpeCB3aXRoIFNlY3Rpb24gQm91bmRhcmllcykgVGogVCogRVQKQlQgL0YxIDkgVGYgMTAuOCBUTCBFVApCVCAxIDAgMCAxIDQ1IDMwIFRtIChDb25maWRlbnRpYWwgfCBQYWdlIDUgb2YgNSkgVGogVCogRVQKQlQgL0YyIDE0IFRmIDE2LjggVEwgRVQKQlQgMSAwIDAgMSA0NSA3MDAgVG0gKDEuIFNjb3BlKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDY1MCBUbSAoMi4gRmluZGluZ3MpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNjAwIFRtICgzLiBSZWNvbW1lbmRhdGlvbnMpIFRqIFQqIEVUCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiA1LjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiA1LjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiA1LjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiA1LjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiA1LjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiA1LjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiA1Ljc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiA1Ljg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDIzCjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDA2MSAwMDAwMCBuIAowMDAwMDAwMTAyIDAwMDAwIG4gCjAwMDAwMDAyMDkgMDAwMDAgbiAKMDAwMDAwMDMyMSAwMDAwMCBuIAowMDAwMDAwNTE2IDAwMDAwIG4gCjAwMDAwMDYzMjIgMDAwMDAgbiAKMDAwMDAxMjgyNCAwMDAwMCBuIAowMDAwMDE0MTQyIDAwMDAwIG4gCjAwMDAwMTQ0OTYgMDAwMDAgbiAKMDAwMDAxNDY2NCAwMDAwMCBuIAowMDAwMDE0ODUwIDAwMDAwIG4gCjAwMDAwMTUwMjggMDAwMDAgbiAKMDAwMDAxNTI1NiAwMDAwMCBuIAowMDAwMDE1NDUyIDAwMDAwIG4gCjAwMDAwMTU2NDggMDAwMDAgbiAKMDAwMDAxNTcxOCAwMDAwMCBuIAowMDAwMDE2MTE1IDAwMDAwIG4gCjAwMDAwMTYyMDIgMDAwMDAgbiAKMDAwMDAxODUzNCAwMDAwMCBuIAowMDAwMDIwMzA5IDAwMDAwIG4gCjAwMDAwMjI4MjcgMDAwMDAgbiAKMDAwMDAyNDM3MyAwMDAwMCBuIAp0cmFpbGVyCjw8Ci9JRCAKWzwyZjUwODkzYTFlYWZmMTExOWMwNDcwZmM0YzU3ZTI0Nz48MmY1MDg5M2ExZWFmZjExMTljMDQ3MGZjNGM1N2UyNDc+XQolIFJlcG9ydExhYiBnZW5lcmF0ZWQgUERGIGRvY3VtZW50IC0tIGRpZ2VzdCAob3BlbnNvdXJjZSkKCi9JbmZvIDE2IDAgUgovUm9vdCAxNSAwIFIKL1NpemUgMjMKPj4Kc3RhcnR4cmVmCjI1NzUyCiUlRU9GCg=="}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"object":"error","message":"Invalid model: invalid-ocr-model-for-parity","type":"invalid_model","param":null,"code":"1500","raw_status_code":400}' + headers: + CF-RAY: + - a34832c9ae37ad44-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 23:54:08 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05f64-fa6f-76c9-8b39-fd7d42840ba2 + x-envoy-upstream-service-time: + - '2' + x-kong-proxy-latency: + - '16' + x-kong-request-id: + - 01a05f64-fa6f-76c9-8b39-fd7d42840ba2 + x-kong-upstream-latency: + - '10' + status: + code: 400 + message: '' +recorded_at: '2026-09-01T23:54:08.641470+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKJZOMi54gUmVwb3J0TGFiIEdlbmVyYXRlZCBQREYgZG9jdW1lbnQgKG9wZW5zb3VyY2UpCjEgMCBvYmoKPDwKL0YxIDIgMCBSIC9GMiAzIDAgUgo+PgplbmRvYmoKMiAwIG9iago8PAovQmFzZUZvbnQgL0hlbHZldGljYSAvRW5jb2RpbmcgL1dpbkFuc2lFbmNvZGluZyAvTmFtZSAvRjEgL1N1YnR5cGUgL1R5cGUxIC9UeXBlIC9Gb250Cj4+CmVuZG9iagozIDAgb2JqCjw8Ci9CYXNlRm9udCAvSGVsdmV0aWNhLUJvbGQgL0VuY29kaW5nIC9XaW5BbnNpRW5jb2RpbmcgL05hbWUgL0YyIC9TdWJ0eXBlIC9UeXBlMSAvVHlwZSAvRm9udAo+PgplbmRvYmoKNCAwIG9iago8PAovQ29udGVudHMgMTggMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXQo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKNSAwIG9iago8PAovQml0c1BlckNvbXBvbmVudCA4IC9Db2xvclNwYWNlIC9EZXZpY2VSR0IgL0ZpbHRlciBbIC9BU0NJSTg1RGVjb2RlIC9GbGF0ZURlY29kZSBdIC9IZWlnaHQgMzIwIC9MZW5ndGggNTYxNSAvU3VidHlwZSAvSW1hZ2UgCiAgL1R5cGUgL1hPYmplY3QgL1dpZHRoIDMyMAo+PgpzdHJlYW0KR2IiL2w2I3BIcSVSZE1bNDNaKzdadVpbLUFpNUtMMzIvY0tJbkxtaixYPmxxXEVxWFItTGs6PWw+I1lObGclcFhybTBwMGRhSFMvXHBpPC5cPyFyU1Y1PGpoNFMhcj5xc1hSVXAtN18zVjItWWksZChNMkNrWk5yLz9aTUFIV0FwJUthNz8nWS4zRzs6WW4zZlZ0TVtFXERkPkY3b2FlZjgrbkw6XERhWFhyMjpMN25AWUAzJ2ZWWktaMTh0aDtXMiVJTV5yWylmQk08RyhULVx0VCNERidTRWktKCM4YTApUiVEODMyazdNQGksOjU1aT5GZTouckk9amVcNE9rW19gLG0oMEI6MW1ZNXE4RURtVWRdazs/KCVIbS1zcFAsMDhuQzlbWy5yST1qZ1U2ZTIsX2JeUz1eNXNhXEwzXGtqQ1RmZkMhWWRyKXBVVj1iSk9janByOG9FJ2ZWWTBETy1KKlg3MTZwTV5yWykyb0kuMDJjL0pGSDU5VDtFLExuNzlBRVk1J2c3SSpERSdUUVhqSSpvI0pUSHNtb2BuOF9tcFlHVS5ddF1sQ2toOGNmMGckYzsiLWBHa0ZpUSppIjUyaWVlXSMlQD5baXJCMzg6bzpFbzkyazY/XFk5IUNiRWhwdURNXnJbKTJiQD07bzY6KD5EMG0lImdpLi8mXVFXWjFoWFBwYy8/Wk1BZ2YvUE5fVl4+PzJYRlJPRVBvPz9ubGgtaVglMlIuWmFmIkxFcmtpOFNBLl0lRFVsKSdpZURNL25saSEscmFZXWU9XjVzYVxNLzpcZzBhJSVTK0o4N1xEXlxnXVUlS1otVU5UW2A+KD4lJF5UQ2JyJih0I0QiZklKP25QYmw5Q3BiXyRDQ0I5WS4zSEcmcFooWD9cOlFOMzVcWT1RKzI/TTM5U3Q/PjpJY1M9NWo6KW09WVQ0cC8uMmxZL0NNMW8lRiJvL0ViJlY+JDteb1dpO2omWCM1PjxjOC9tPlhjXVs9LGExPkRpOkRnZk8nRFYtcWA/UXI9O1pQXT5fcmpIW1pdNWxyJTBJLm42Q2VYaS0oPTlNWiFFJnJRQWdFayI4a1lmQ1Iyb2M+LlMlUTcrazlZL0NLX1NLQ3NibzMvLS5HX1A+KlErLWdzO2RNZT80bHU9Tj9aPzxsNG5IZV5ZSHN0M3BGR1o+WSZjQT9yVWd1V209KiJwYms5O1hyKj8+J2FvLilZUihHMjdub3BrOWdBO1YvMFIlWExqbVJBOC0yL3FYQC4pIztiMjopPE9uVUJvQC4pbjxha3IwdSFWXCw0QC4qYTxha3NsaG9Cbk9wWUlNIXFQQmlBLGlLXms6PWtyL3BfYiQnLDhdPFVQPWtxPFhvLDNiLFdWKVNPLzhlWzBhRTVPMjVmbl5DTXFhR0JxY3FmI3JeS245Yj9UbDNpIiNPbWNMMTNdViFvbnM4bURAP2BvZWdwUSt1TGlwUmBzVSFTZ2UmJGhOKGhyWjdCX0knMUJrRTVVY2hiXEoka0haQjpcaz1TdFoyYWNIPV1RVTNmZ0RBN1xZL0NLX1BOND0+Q1kvRXBHOSZqMDhtRD9UOllydCRHLWVxJlFkVGlVYCEoKllqWG5cbFFpIkNAMjtSQ2U+UT9dZWhTUXBLXlY8RFhlN3UvKSJgT001KD9XIjpycWBndGNXbDs3aSp0YiFiXEwvVEZeQSs0XElvXm9LdW1QZGYqOyc6XFp1XWQscWFfK1BkYD9NYzE5SnRJU1ddNjRCJDc5bmVVQXFFVzAuYm10cClzVy5eKHFba1AhW29baXFmOkZpLC8tLz5NSVFiVkxrUSw3PllRI0krTD5sSkAlNHBrNTY8NVBjTmxaLktRMCdDOFg+JCFCNW8mMDMyRWpdUUtsMmZCcT8wKjwsPyskXycmJSY/Y0JTUjJDIz4jbEwjTER1UEY5cloqL0taP1dqO3BQM1lOUVRJU20rQlhnNmJTaWNVUidjUV8mbTsyVT1MVHItZDpfbmRhXiZoYV1McD9OOW9rUihTV21ITjNwTzlwMF9ia048Xy5hKUxfQ28sQk1YaEJkWFIncVhKYCQhNUhmbidORCs9OGUrVkdgNS09Z3NtcGU0bXJURCxrJEsmK1VaUWJXLl9kJj9XWkZINl4tU1pvW2k3KUljL3RrU3BRVTc0Ri10dGY3czlScmk8VFQpai5sdEgvSkM0Jz5HWFA/RVg7XGo9YTxjcFxybVZuIypkXl02IkdPUVJENWxJcjNkMl9xckk/bTlma2BqUnI6YEs+KnVWM3BvSy9uczpgLWRwSiM2cGM9SS5cOWpkaT9fNG9aSWs9WFldMCVTJURSVjNJSDFFVGBDTjBcZ2pBPzZYVWhCTUhQQllQYSRXP2E4TyVRcDslRWkrTV1HcGlSS1lnZjg8NUxmZTxyVltOaD1LNllhL20kRy0vPVRTRDwqP0doSkljMEN0UDInPDVLKixwW05CUGsiUzouaj88JWIlUyFWLTdqWnE9R29wQ25AR1lmVzhfdWdhRDBQTCdIO3FgalBrWF51XSZhKmc4Yi9KTUNLb2lOZSJbT25YTHNUSWZJQU5ZOXAuSFI/WCJWaEwpdVYhX3JaO29icyREXUJUZyMqcm4/P1tHNFZZY1JqJTdcYjtSSiM3JEhbQG9bZCtKWkJbcCtnYSUuZT9iJk8xMFVhOCpYTTxtPWI8P3U0RTpgLj1JLj9kP0dWaSJLRS4/K2tTNHUqcWlNT2NwcSJkKjlTLEE5QG9vXkNlWyJDanBnaF82WGJOSi1DakRXQ2BHXWtSXGRrYUpNYTYlWW4yWWU8ZCxtdGNKUjxzXD1jSTksQWhwQ2dQSmRsU2dTcmwxVz9fNHBgcmBWKzM5NnBlJCtGZVwtIXJUSCQkS0hLUUI6JEE0bGdfT2UnRUdMZlEpI2JrPHBQK0tZKyYvRVdrQEFNZVR0TkJsZk0nUzBeN0xyckpUYF5pPSM1U3FtdT1ES0Q7JFIxSHRaKTkhJT1SJUBTT3FoSHFjUkRhO1hWbmllK0ZCOl9pczY5WWpdKVkzNmRiTjQuNyI0Jyohci5XQUksOFk8WGU9SmxPX1ZEP21qaUxgakRpKT5WNGFaN1ReRyVLOWhIR0YhJDdHTjVQWlJrJGNtM25bckFFcHRAbitlOW8/MUpwUk51ZVZJZ3I+cUdnRlshYj9scmwsXT0/RyVBZSNbY2ksYEQ/U0lvb0RdYUg8VTJYTWQ5O2UmaT9vMUkzPHJROjpfR05KIlNsPlc/ZDNPIkxxLm4oWkxcLzYhNk1WIVlQTjZURVlAVDIyaFhmVVdqPV8/RVAuLkAoUURYLltjZUdAXF5ML2xpaFI8RGVObCEvLFQ6QUZOSyRiaWg0OFdlIiQ/LFpkR1M5Zls3WyluTW1dVCYwXVt1Y04pPDxGZjlwMyEjXkQ8aj03YUU2InEkVFpVakxyRTtNYzlCZG1gQ2QwNEYobDhmLixWXkNaImxxXCw+QDBIXWMndEkxP15OKE9YIWdRUkQ2LXJZaydDaSlrOUBwRkNgPW5tOkxfam85UFdFLV06by50O2o4aFdWLlNyXVIxZ0gsOTUqSXNMVGtsRUI9STRsZGw2Xz1vSTBIMUtoWFk9OEpANV5EcFBtb1RNWmpMczhTTXUzTm47cm4yP1lQZE0oSGI3JC5uVHVzcWpUWFBGcjhsJF5eZ0IyIzRdREUsXz1vSTBIMUs4TWY2MldTLFh1LDByUThLPV1PVWxJcCMhWHFQOEFWXW5zOmBtLF8nW21IT3U7MEsndWU/byZMaFJZUEFrOmI2dSRrcTkrdEdUImpEYShiQD84SWNXNFloTkBma0pcbWM5SUBtJ0NTa10zRW5DUClfXixrX1BpJnBVSW5vb3UrW10mXEddUlFkP1JISVguYWExVmYxITsiYW9lQF1DbHVaRHFmKGVSLlA0c21tNkZDLk9tLUU3Im4kJG9aN0luK1xWPDsyS3MnRDtrJlBcRjRZZU5qJUFkPVs7P28xSTNXVyRlVlMhbVheSyRiaWg4LEJoTERkSWhlJFYkP0BTYGZIMzRXYm5dMl4hVWRCVGckR3I6Uy8uXXRhdXU/U0lvb21qKkJHOSJvPi4kSGxkMVJPJT49SiI4QG9pNS0xb15ML2xpaEFRPWgyPmtfPC1uS2thQlRnIjFyKjlzKU9VRnVmcmRyTkNII05HJDxQPy9XZmE+bzAwVWE5VWVjJ0dXcFAsMDtZbCJbIjhiMWEobiIrRmNcVG5QKj9TSW9vbiJwIyRNdFc6JlVwMGIjVUlrJTVrak9AOypjP19MbmVbNjNhYTlTbkxONURjaTxtYjgwVWE5VT8yZ0YyamxhVzUvOllLTFAuVlYsKV48Y0UyYypyWSRWJD9AXXVBa1dRIz5RUEQjKTg8cEJsQ2NoWnJdP3FoSkRUbyZUMyxEaG1OJm8vIkhwMFVhOCpGKzddXTRtRGUzLGs7YjEtSFkjcWhmKmlPSFRqSE5xdXJpZW8hXDNNNmYuRitbbmU8Il8uXSJFNG9KKjwybCxXTVpjUVEuUk51Zj1KK01gXGw/VkVDbmVbNjNha1AodDwtXFJtPzlQNGEiZiJaW1tkbEBdaFVuRWQ6K0xoMUJUZyNWcXV0LlxoUiE3K25lWzYzYWtPYUBONF4jazQmN0pZXy5dIkUqVzFXNVMoK2UtcjoxWV8tSFRLWzAyZ1o+U2owKjZeTC9saWhIR0VGbnR1LUk8I0V1YXI4bCUpa1YkOSFtJS0/SSIxamM0NEYrIiovV2dqckYmZkM7QkFmMWpQSj9vTDxdSy9qVCQlazdeSD9mM3JtbSlnVldaWmxucz0pUz84bWZINlxBSEtkX1cmI3JFY19vaylvQ2I8VFA2LlhPY15TPlc4LmJobEEiWzhbbi4pYW4qQGNqUz86WUgqRl9pSS5nJ1tAPG5vInJkY0U8JGgxV2RdTiUrSSlqJWZzSCpkSGQtI1s1PGE8J0NxOllvJyM0XHBSTzRpOyR0bC5QLTljbGRWcFY8U3FxWWwqVilTKWdiTz9FMFQqJEhsZUpdR15dcDdeVXNHNj5qNktuczpfQlxBLT5KbSwxX2EyY1g7XmJFRm05P2lPNGRuWWhQPVZtLSpcYVcnSTRkUW1OclcwIUxFWWwqVikpcy1QUi4oOF1GT0lNckpyOGwkPmlwWGt1RzRWWW8pbTZGQ2ppTWw7bikhZ3NvVmRpalgwRE5gYWE8Nz9wPkdXOVcvcXNzWWwqVilIZkZNOWFMMV0+TztrJSNyOGwmNG1JMCM8Rm5WYnEpa3NTN2ppUC10bitpPi5wU2ExI2hRP187YWgtTU1hRXA9KU5zYEZRJUVpK01dVzskJDRlbDxBOydHXDBCREtgN0dLSl9qSCMqJC5fVksqNnJwby5DcVxwciVEO0wyXEhfJj5lVCJJZ0pBKidpO0hQXm4kSDI1WElmQWdlWl1YPCRnajAmKS9Jc0xVNm9KXS5vZzpUQGJRWW8rZ28mUDVmRGolY0BHcVw6VW1tQCRFakYtJyxUIk1JVm4pWlRLNy9sYzxyUUYodEgjKiJYYzFUa01yOGwmdGtcbW9CWzxaXVJEVldqb0gyI1oyR3JQWjc0bXJCUnI7REV1YWgtaixKQG1cSm8oUDs5LGskbzIxXC1SYmIyNS1rUC5aNm5hNWlmMFErcWdCOGIrcUBTV249OD04a0NYOGIrTTRyMmI+MmsqXWosbz0nK1NablwlXFNyTGtMbnRGSyRtZF1vSE1Dc0goXGBNa0lOU2ZCT3EpPFdoLk5dcidXYS5jNj4sU0VLS1g+Tj1vYC9hbzAzcDNKZE9lUSFkJTNsKkdjZW1TMTM3NyJcdUtENHF0WDU2R3FqVGAvXEgqISluVD1yJGNqamRwVEQpT3EmP2gsOklDPzpNWWtPSjstPGZlRDxGRCxBK0dOU1J1Oj9NcmU+N2hrRSVMak1ZZTIrWEtfVWNoanU+S1QuPW5pPmJvNy1wTWAySVgyQTIhJGZmKTxRRzRnJUluYFoiOG1FMkxVVGpbUGhGZVs3QjxLMSxFPC1FaWhuP15MMjZIIjU+U29EKEBKUk8/Ml9Mcl0+X3JqS1ZDRi1IPTgxUW8xbjYoOUBHRXUham1VWFNeNS9BJSxxYi5HVXAwZGltdW84ODQnMUZZXC8+WHM/ZnNVKF5ba11lJGZjc2wvOlRoW11fOksvLHFiLHFYMEROQCddKz5CYC5gMTUncGNWJVdpcWQ/RUZJWVdrSzddM0dKTHNAYEdOZjNxV0JuVG4qYzR1J1FIM1VuLDZNMUhNXWVkRiRhbDlFdDlXbXJlPU0uV2I2PktGJjFwbXJXXFxBWVwuc1ppLDI+XXIhZlZbcWdRTCljZVknUnEwUTNkbSFjbm9jZVhzUnE3Qm9RcmRNZyxjWCEpL3E3QkwtaVZlV3FUIj9oPW5lV0VlQEYqcmlhUyFrKVxcQXUsJFtXWmA7UWInclJKaGldPk0tOWxwTXJGZzhhQSQtaFNQbCddcGVoPDhpY2hSYkY8XmppZjwxXydhRWovM2UtXGpnO0xONyhXUStyJSppV1dyZyhXQDZnJSdIKHE0NnQ1PVMqVCRWUWpKPTVxKixoJVUoZj5UODFePl9yaktbU2xsLlxHVnVaQmtfXXJLVGYoSD1fbU4kUlYvWSk9c1UvcDwwS0V0QC5LMnVfXyVNSm0oREVcSXRxVSYucmYiMF9haCFXVVx0cG5BP05rKW1EOm0wYk5qRjchO0ZccT01cSosPD0xLDhGWyJPPkhQVF08LF04SVUpMHUxUWI+SSohXER0LEVLdTxNLiwqX2dRWS4uJi5ZZW1ZalNWUVtcWmFmIUFFS0xTNWs0S2guOF5IQ1JTJkMiMCowa2E0XmpVTUU9NTUmUjdjQkhBWCUuJFdaYWYhQUU8LUU9a004Ky5cIjpQUjJpNCQrbjZ0MWVqIy0xKz5dR3Q7Vz0jOVloKityZVQjREYnTTQ6RjRHMkFFLVtJJSslRUtkbigkTGQoPjRBPXEwYD4oPWdTIl09QGhYYjU2bW9gbjhQSVRRWVlOPzlTNWxbRnAnZlVzZ1g3MCRELi5lNjtNXnJaWDJjL0lySXJGLGpaYWYhQUVGW2w1QENZV0woMEI5OkRXJTk8IS1FLyppZWo1RE5WWSVfJ2V1QG9FUShaMypyPlEkUmolNnM0YzpoLicpMD5NMnIjWyhuKk83X0VLZG9lWTVRPmFQUD1ANmA+KD1nXTRdbkQyY1xfaEhQVF08LGpwRDJeVWAlZyVkNyYjRUtkbzVwQU5uTmpJT10pXERlaVRRUTFEN0BjU1FZWmFmIUFFVyU2YF9oSDJvV0ZwNGk6I1wzXnI4bzNiKlMuUixpWy9ZLjxvVzUzZjxgIUgtPzI4XU0zaUVzY2YwaE9gOWVvP2U9T1ZvMnNMIT5hWkkiRkVRKForSUg2c1RaYWYhQW0zX0NhbW9gbjhQUC9vcHJ0Iy1rPW0jfj5lbmRzdHJlYW0KZW5kb2JqCjYgMCBvYmoKPDwKL0JpdHNQZXJDb21wb25lbnQgOCAvQ29sb3JTcGFjZSAvRGV2aWNlUkdCIC9GaWx0ZXIgWyAvQVNDSUk4NURlY29kZSAvRmxhdGVEZWNvZGUgXSAvSGVpZ2h0IDMyMCAvTGVuZ3RoIDYzMTEgL1N1YnR5cGUgL0ltYWdlIAogIC9UeXBlIC9YT2JqZWN0IC9XaWR0aCAzNjAKPj4Kc3RyZWFtCkdiIi9sOTYpO2cmT2BoPmVYVDgrZFFNWkZQbm1RMU04ai8kJDM7ODVOYzljbTkjSVk2JWc2MnBGJS5ybT9YTmoycWQyU1I5MXFvTmxnT01pJyNVYStVUiUyXWQnPzQmOkhlYVJEZDdfWi1eNWdYXERlUXFoJiRJbEJcWzY/YEdOTktQPUQnJ10oVDkzRVNHRy5VXzBEX1FAME5NZ2JyRVwvJ3JaZG9dcmluaWpSZkxEIixdbzwoRjhZTV5yWmFZMGQ+OVxTOj9tZTdXZD9EOFdiI2kibj1CKkd0WChZLi8+WTk2XkUkUyJQY19JMjVvPipvKWA2UzNxMmRTJ3AnZy8/WkwiXDlqU0AqQi1xPD04LGomRVNHSEFZUEFFQFMzJCoqWmFmIydpTFg8RGNlS1hjPmhXaDFgR05MVV1bbXElXS9qaXJaYWYjJ2lMXHE/LE0tITtmQ2A5T0VTR0c8bV1yRExXL2VMZC8/WkwiXDZjWGhNPGJQX1U8aGhWZ09uaGhZMHUtLXA0am1eNSlVcS8pbTkocWk3USt0XidGUiJNXnJaYVkuSE5eL1Q3MEoiTUUoP01WRWspK2NVLWQpaSs7Vm1eJ0ZfW2MnW21qIVQwTWoqTV90bUJhPV5bYyw0dGojUWhRaFkmWiMvP1pMIlw9cT5xNE03cig4XGROSi5ySDtEWm5yZ09EbGRna1hqPyFhZ09nSTsoQSZYJGpmQGBwYHAnaD5palJnQzhgT2dgTDQvNS1JMjVvPjJWXnU/NFBucEttTjguS2llbEtGUmswT0heYDFVVzNuVl9aPl1KVDFhaT0rNE5xZDRFM19LNiJZLi8+WS0uNyNeXiUoMV9cIillbldUSGguYG5cSC4ncUguNSpHZmRDRElbU09idD8nOFVDTzw3JyI5QU1zIjNISGE6KFdeUl5qJz8wYzArKU9sVSlALnJjTUwmTURKV2E+cD9tZUV1RWtnckEuP2xnbCNUJTlLJTRwS1dSMFIjPkk1P11kLnJaakElamRhSCE7XGhEcjZYakdNJS9EWk90OStCOnNxUmpXNDhDYlVUbmwmYjw/PypxQVFLOmhLPFNmLCVeJCc3KDFaK0Y/Uk9gQHNGLGNOPGZjMFc7IU02TClpanMzdWY3RF1yOT4vMiY/WkYmLWs6VSRDSCI0NmxqbFwuc1xGLy8mcTNPJUs+Pz5qWmllbSdJaDNgUzhqMDQqJms6PF5GbWJHTERRVCYxJ1F1S2UhTVgvUkRTcCNDc1onZlVwKWsnaTU2TE5lbzRhQTlIWCNQTUM9NTtDZyspY0VMWS4iW2IoMEI5dWpZR0taN1dhRSRxXmVuWzVLZGk8amMvLEYoMEI7W0NATSxsRiw7aWcpTGZZMU07K0VQaipmYSlQckkiQ05eJFYpMldqai5VRzIuWFJsOmhIJW1GKFMyV2hTRFVHMi5YVGY1YCwhLXRgY21FMFIhWW4qI2xcXk1NUEI3VydFaVxvcj04WVwhLkYwLkNwOic+Tz1gLDNeLVB0IzM9a0BTWEtTLiIlVU07K0M6L1dJV1tjZk8iYzRXVUszJyI5U1NnKS41Q1REUT0sRkErWjgtI1I0MU5caCs3NSRiZzA0PWxUKSciOVNTOWNBXzJUMTdcQSojWlV1TTsrRGUuPzIzVzpHNldLQXAoSyYua1YpZGlAUUhRaFBabydUJ0g+dEZhZFk6YklMVXQtWkFicVJRQipoJkpfY29mXTNNOUpsMmooNGVncmVOZ1tObDspbWshXE9UV1NvJyk7cmlpO29xUE9qWzRcT281VTVrWDtsXDxoNlotYFhXJCV1Myo4bEBXdShPL1RRMmRPPUpSNmZYMChvWEs4Kiw5QUpfKW1lNl41PjlwWXEjM0loXDtsN2lOUFtrREFEWykqcXJkZ29vZFwvO0ZOU0FlUE47WkVidVtFTUhVUGgxOHUjKmU0LjdHYDJgRmtbamEiaSQwJEJjLGIuRU5YPmZKdXJadFBkUTZ1MDpgXk5jXFZfbD1iYVhpYVgtSV09P2ZIPkZgP2FgSCxzRFlYakRtT0leTzouKDYrclAxISw3RmB1NGBCPD1WOGIuPm1RNEtidV5Ga3U8Ij5pXiwuQ1hbVTxRSz09NEBjLzxucDVTVjlqVmk+UGU9OFYtSFlUS3FhUVhaaV1cRT1JVEtpRzFlRCJERF8uamkyUXNXOjosdFYoalJyMiM+b2cucGFnbGA0RUpQc2BsZz40VzNhaFItRV00SjNoS1ZKUWpQQj1GbjorXU4jQWReQVk0NGQ9TkBDO0FuIilJbXM4LnM/QC5QIk9tNy9VRDlbUCwyXCVPTm85OkRwSVw0X1VVbzo7VUludCgpOXBmRUhNImUxajldNlxZa2BZOy4xcFdzbEpJVi9FZ2ZYcz8lTkRsJCJWSikuKjAmQEhFZl1RJkhqc0IhU3JLXGpNOVtQLDJddD8pdGFVXUpIYy0qb1lQSyoqRT02LlVvUE42bFlgcGVjQEkyNF9aRHNtN0o8bU1SRURhJ1U4WUhLbT41PG8mTWZZITNTTFhJI2ciKjM2MjFlRCJ0RFtcbCQ+LU46YUVtTFI6NEJePUxha1REQ3BxJkddQ1xYbFFsOU5QSVAzWDxPaU5aa1NQckhqckhOUTIoaSh0Ry1IMkRTZU46SD0wIVY9R1EjaShVdSloLl9HZ0RdZkhXLW5JcURiU1AoSCkmJi1yLltPXF8iS2RfZ2NJMC05W1AsMic+LSd0T3QoVWlwPVssJ0g3aiwwZDNPIVldSjk+VSZwOyJgZnMuRVZHISYwR11FbTpfaFk9XSYwNmw4NzJzQFNMS2YzKWpSJWQxQ0FwUnReRDk7J25tQl1HU1tvX0ZSVS5PbyZIWGxpKVJIZj1pbzZEVz08bzA0PDhiMiJqQSwpVS9sQEE4Q0ZBJTY3Q2pobkw4Yi8wbGo3My0zT1cqbGZVbydeVGlsVzM9MWVIT2pEajcxJlthM10+T2JVTjJMKmsjSGxdJDo3XyJOJldONyplSGRhMDdiYklvVnVgMk5qSWNgOHA/Zy01VzxeJmlSaU1DLUgnaS9KbStMUHNYRVY2W1InaEZMLCJqOT4obiYvOVxNbUJhS2wpQzk7SVVJayU5R1tuZzhWby81XT1LPSQ6MmoyUTZIKGhsKmA7KzlOW2ExOG5rYCEwOk80Ty5HclZfYGhCYDkoIiguKGY9JjZkKFIyKEFtYCJEPDotS0RaRmptJlApKTVWPl8qX1dfMGkmX0lvQUY0bGZEcWcpWy5uJj8lPlYxJTNZNU9FdT4/OUErZzUhLEsxbnEpM3VcOktocV89Zmw7XEBkcl8tSFYyN05TdHFlRzZRYWFUWkhbbjE3QktXKCNCZEYiNXQ6KWpiPEInXUoyaj1qTHJNT19rISVsaEIlZzJuWSsvaVlJbFNZUE45RXUpdDNMLFpwISYqXTsxQTEuclMsLyxeSFRiYGMrVEUqUSVPRzBOViJVSCIyXllhYTlkQUw6YiRGU2cxJmlUIXEiWFVYOj80MEQzPihBIVZORXAzQFM7ViElOzwxN0JLV14ic1VvRDdXclk1KTJiSjRnMkxRNz9jZkdnKUlyc0lHUVlXVDsvN0FTbGFkOVUuT3E0bVokMDg1Tzk4Y2NPJiQ9Y1JXcC1kM051VkdbbjgqR008PmRGK1EsZXFPR0kqYi5FM09ha09dUG4kYCkoRitjYFQ4XSpdcF1KMmo9alIpI2ZpR08tRkgoUW1oaVNKM21AdTtWVGdXVVZTJTFndWRkY1kuVFxHJy5fUiVkMUNwQEg6Q2NgY1YpbWRrTm4mXmtYZVpiLyJdUE42VEZOVCU8ZW1PP2slakdkXCw4YCRXLCkiLCUpKk9YWGAyajJRNkdvTSo/XEc5WlhQXSZoRDg0SmcobUhvbmxybVBhO19KX08sN1BeRFhEZ1FFODkrY1QrJloiQ0JCNmUhRS1IU3A/TkZAcFlNW1VkPXBISGI+O2AlJ1AqY0RJbjZxInFUTDNvUVNOQEM7QTA5bkpORT8qRyVeSD5YO21YOkJbVkopLipDdTRxQnInUXMjKShmMidHZVNxTiFSRmooNWlEQS03WUVnXWRKdURlX2pIaFMuclVZbj5DKzFnKkhsLHNUJ29TISljSlJiPzxWN1VVTVlaQUlLTXBubzkpJjBCVGckUGhLVGpgVCVFc0VTQGczcURiU1AoSDAoI283U0RnPTpJNCp1L3BVcmEtSFJjVl1TbFNHcCQwa3Vqa1ZHNEB1O1ZULnJUZiszWGU0TlohaU9qbzU5ZiowaCU7ZVByQ21eaDVIOCZZKlFeVDc/Y2V0Mmw6bEhWMUcuXFdTJGNaVExuXnMpalxSI2RLJVUvMDZxJmItSFJjVihmUD5saiwxTGhPdUZxKlhmcCw6PSc8NTNdT2Q8c29ldUNXRio9W143NlcyYWBtNydwP2QkME43bigxMnIqJT10TkpsbC5Dcyg8KiVEQ1pvZCRFTzYycGBncWpVRmxERipmJW1bP25OXDc/Y2V0Mmw6aytlLGMlX1dTJGNaVExuXnMpa1kjb1c8NHNGQzw3RG4xZUQjIWokVjcnbjMobW9rMXFHMkB1O1ZULnJXKFlxKVMtNWNfS1VYLnNZV14tSFJjViRvOycyMDc9MFdtcmxlbDtgJSdQSCIwVmdYLlJBUUhXJ0ZTJ01YYVdmYyw5Vi5dMy5eSUtNcG5vOSkmMEJUZyQ4aEplVXNdLCYoXEghWmhVUDtmQFBQLls9QUUwU0FTMCJrMGdnQSlmMGAmcUNQVVJeKWIpdCkuIWY7UEIoZCpEQT4sTi9DPylbUDUyaE08az9yUURtYz00THIvQHU7VlQ5QkQvJzkrQF4tR3A9MiRQKmEzLlJGYDxIKlYpWlRSTnNQQzJ0QVVXTGNYVVZBMkhtN3A+LjNGa0NCMzhHcD0yJFAuLm5aJERkZC8hSTVuLTlwMnVqMnAxUz9fRF5KN0EySG03WTIwL0A3ZXBaV05GWGs7aE5tI2M/SUphOD0vZVJEODRKZyhtSDRxOjFvZ0RCWTc+aCYpLDY6J1I1XilRLi11I1wzb2w5MjFlSFAhaFFVNlZLS2ElSzZmKEAnUE47LGklSyJsXk8/VFNBNmYoQCdQTjZUPSVEK0hpcVJqMVBLS11xQFNBVkpPXTdEPE5IYjI0O0RdX3JOJ0BcMUQsYm1zJT9MPGAzSFxeMjhCUV9COGFfNmpNcE5nPzxcY01MYjwjdE8hMTdCS1dWO0JrPUJHXj9wVj0haEMxN0JLV05TWlkqQF4oOEViYSJBbmhPKCQvbm06TD1fb2k1PEI6IV9mSDAoIzdSTyU+amhKZGBOIThELjViaWg1RG1IcmFvI0dGYFtoVm50XEJkIS4xLC02LHRIIjJeWWFhOF8jQC4tcmg/SDouWDdQXkRYMD45QGJWMUlGYlRdI0IxMTdCSUFcMlBDZSFUKiJRcHNUX28pdTQwTFlMM0M9XStBOFVQTTFgPiNNTEVMPiRZLidKb2InKyguXkMqOkkzMTw4aychRlJRckhYVCcuO29Ncl9FYmF1dDZOWWYmXFNVOysoXVFKMVBdRGROPEQ+aSJRVjFlRCIlRGFbNSNcXi85IUQzO0puTiRjWilqLEYxVStTWFBxOTkjKW5QMGtQIzYraT0pJUhLXStRK15KbzdQXkRYMDdsLSdNdWhldWVNPS1saEZMLCJqPVUqcVZET0QzXU48U01hVyY/OFAuXDNaODxzSSNUPTwiQk5aXSo1VDhkTkldUzU0YFFOaEsmOldmRGwrOnFQRi9TTDxbLS9MPT5Sb0F1P1x0cVwsW1FbTDcsXkshcDsoKW43NStibkNBKiZcIVJOdWdXMmVvMT9naDJ0MkRRZ10tNz9jZldvTD9BSk9TZUJFVGhVaDNWSikuKkMjMTZzIydjYVwubyUsVWBZOy4xMm9jaltkUW02LEhVW1IzaE8oJC9uZzhTYDdpS0hzTltYWUliOFhBOjhgIi0hNVkyJ2djVSRbSUlzR09dLUhSNGklRWhbLkAsPz87N3MybGw7YCUpJitFJipuIUxVPkJmNV9qWGEsNjlkaEMuUSklJSg8PUo5ZWJJPy46JDk5W1AsMmg3WzZlOlk8PT5cYGA3PjJsOjhPXVk8NyxPIVNXIWhWOkpAYTtcJjc4Uj1vKi8uRWtAPzNdaSszb2w4TFJPIyknMmNgRVwyRDVBXDpFQiNHJ01YYkJgPl1hYCNiLjA5XTVfVFZgWTsuMXBXc2w2Y2NgUDxXT1QhQS1aO10rLF5FMXVZcTZGR11QUUIiNmUocCxkM08iSF1sPjUqakxnLk5xYmJLVDtgJSkmNGA7PDQiJCYiXmY2RXRaYFk7LjFHSmJBVGBXND1YYk0+LWtIL2ZXNlAxU00vXCojKCNhcj8rTFAuXE5jQlRROyNIL2YkJVAxUykjNG49TC1HVD9RY0dvYjlYSCtnbDwtRWpxLzkmQTJmUE42bGFJNyRHKmY/SkBuUk8lPzdEWVtEN25FWmZvOi1iJChyUDEhLDhDXTs3YEI8PVY4YjBVVVFCLmQqWG9iNlk5SmwsZCslIk07PEJbY29RKWY8MTw7MEdkbjdwSyklPk5Ic1YkWnQxOD1uIz1cWCNIUzNvcSojXS5BJE8uSVJuPCxdQU1yYjRDTDs9Uz9CbipkUi1kUC5XKmhXNkhpcTksWjotLUVlIlVQLipndEttQ1hbWEsoNFttLjgtVU5QXCYkSCZpNkFCXjFbPFU5IiF0XUE9Yy0vcmkhUk5SMC4+a2hjJmlPQlhuR1s9KkswXVJiJT9tSShXMVFbLGEqZiYiMmYpSi1FS0tcWi5fT3VNLyg3aUZDNWBAXiciTmdbUT0+PldhPWE9RC1PW3JDIVwlRUNmITdXX3NWYklMVXQtWj03W29fPW06J2MyTy1VKDJJZFYyV2k5UD8yIUdEUik9YU5Vc0ZxPW51PFpnUmFuNiVYSSo9Ti8qKkA9NXBbT1ZQdXI+ZldNKDY4cyRScltsbFo/KlYuM1ZYaEtiMlI0RTozRXRkRHBhcjs7LUVAdCgvLSNTJ0lab1s5LWVfMyhzWygtZ1k0Wj89XG8jPVJFKnNDTSxNOytVQFZbZSFpPzcvNm4zRVAycUUkTmpVRF0lZkVxOGRWJ0RMOE91M29nYENTW1RWZVkvRzInb2k2ZFtFUywsPF9zdW5eaCJGdUNGaVlMTElDP1ddTVJhQS80bm5ZNyM5UzIvcHFEWyNBWmlzZ2NaTCtBR3BCSyQjXm1JaWd0K1MvRGc+aWZYKGdhZFA/MiFVbVwpXSZOUmBKKVQ1U2dYRUxGQWxkazVSS1puX0UpYD4oPWZdL1Q/UGBhSnR1YzwiN2pLPURrYTA1anRoRnNyP2U1KVVxb1YsbGJkaEZOX3RAQHRMUmYydFpeSl9WMilkTz1JZ1xEbDVjSU49ZmhPWD8oKTUpVXJaO0FoK2tYWitbZSZUa2JITUFzTEFXMiYsZS0tUC1cL001OEU0N1swZCteLyc4VjQ3JWhlLVtjWnFndUw3Qm9iXU5EYUFIOTZjQmQhSlFtQTc1IipcKWVPbms5NF1nWE9rRV1KSWY+bjdAS1tkXXU5JjAsRktNMXFuMS5nP3JGIVleTjAlPG4kOXB1WGJCQHUnXi0jUyZeSmldWHRRM1ZkSGdPPzxEQWArY0YuVnAhaT8hKjkjJyI6JDVuOiZAPFNWSCxhP1pGJG9pYDwkJywtVSwoTXNZbzInZlVxUVUoNnExRyVuMU9ZSU5RckVWJihQcVc5WydeViVuUE1WRilaWDpFOmBwNzNIIz1eNXM5RUFRLyhRJDZyWytGUCMsZ2gsPXVGaCRMPGgndTBsYD4saigyYUpKSUdySVg8ZW45IUFEOHFROWEnbWk8bS9jb2A+XUg2UTZlL0g0cForbVFaZDdXZ2dQZClMTmtOKVxZP3RdNy8/Wk0tXDNZJSoydEVSLyJNKWs8TVZGKC9Ub1JbOS9wZFRlWmFmIVFqMTs6JC5CYU8rXlgobnNnUGtJISVTKEVDNSlVcS81SGFHc3JGcmUtcUNKZVpjKyNCTWZ0bjJFV0goWShgPixqImZCWDE1cEBpSnJqMk85QC5ySTAyaVAjNislP2VgLUkyNW8+QiYmUlhyZi5wRk5JMU9SWS4yYGRnWGdobyVAK3JgSTI1bz5CJigpQ3JsUFstN1k8bV8uckkwMkwsc2QnJztYJEtjZjBpOk8xVE4uSXJdVitkQUc3TE1WRikqMm8lQDI5Y0JUIS8/Wk0tXDQuIVZlYnIuakhoOkhHRUxVciIyVFY0U15IUlErRERTRmEkZCRoMWZrNTxERDhvIy43YjloXi5ySS1xTlxQa1M9XjVzOUVMM0trVjU6I2JwJkZdYk0jUkpsM1NKSH4+ZW5kc3RyZWFtCmVuZG9iago3IDAgb2JqCjw8Ci9CaXRzUGVyQ29tcG9uZW50IDggL0NvbG9yU3BhY2UgL0RldmljZVJHQiAvRmlsdGVyIFsgL0FTQ0lJODVEZWNvZGUgL0ZsYXRlRGVjb2RlIF0gL0hlaWdodCAxMjAgL0xlbmd0aCAxMTI3IC9TdWJ0eXBlIC9JbWFnZSAKICAvVHlwZSAvWE9iamVjdCAvV2lkdGggMTIwCj4+CnN0cmVhbQpHYiIwUzk5LE4tJi1VQFxFIXAkZ3BWMjVzMGFrMXRuNyYxL2xDZkBjJmMmSVdZbGw5QUpWaz88RT8nSVdzI003Qj9BUzlPSD9qJUZDcWg3OmhvRkArSiVYZ2xMUk9Cay5uPTJNJXNITUo7bFFaVWdRNl4xLEZvb0BIMUJEVVM6dUdyIkIsaFBRS1ojJixIMyVOJGYwKz1ARlNlV2peTi89MSI4Ikw8RTs+KiIvbEdhQ1IkTW5nIW5pSyU2Z0QpV1JTJkBEKSsmXDRSMForQXM8KnMmcDcsY08yYlpeSz1baGFkbTktKVZmKFxhKEQsXzxXSSRXbUFzX1peVz5kam1WSDJlZjZnSGhPP2BpOExnVlRBYDxON1o7STIiLm1QPFNzIVguMiYpNDJbMypjSFwyMVNKZktpQjI+QTBjNlFxdVphajE0Qi0rQTlmaUc7TDFETGVlQ2FgSDJSPm0vR2VsRkJhKi5TSD5RNSZiMz9scSxnRFdyXTBPbD1UTFcuPzkoazhEY08mPF8qWUM6RkpvXE0jZ20sMkpZRDw1RDJrbXJLcj1fOU9kYmwiKEBZO1hLQFMpUHJsVlFNKEQ7XSo8MWYjQHUlXFR1Mi9wQ2wrPScxMy0xaTwwRl5UTEgxLGUkV0A2ayM+aVY0NU1LPkhGN1BGPW84XCNBOyttIVBGLCFVJE9WYCwqaTFRMUhZKCFpYGB1LjJxXVlpYVM6MjlVNGdZIzkmXG5YPV1XM3RmLEIqNF5gclNQbihDPjEpQVNbSHJPalIlbyZJTjNNbXFsKG1aREooPylFN2pSRVRwPzUkU14jWmFAQ05RUEgsJVtQSXI9NUg2QyckOkIqZSZRZTk0OyloY2pyMSdzZD4hbTxtcUYjO2Y3cUIsZEU4cjlcO2MxTC4+WUQ5MUZIKE8sUXFYNyEzdFpRRT1GXXJYIXM9XC02JzFEWCteXW9GbE49XGNZZDFDbVc4czF1dXNBNiRjT2s4N19pRmcqWjM0Y1hmdUYzYSplPzo+Jkw7TTJJRDlOUyxgYmwvVGQtXlFFS21NcGFeVGhoaDQoaWpyIW9fNW9NZEU9UEBXdTxrNTM/PitCMjtDM19XJFhDUTM/PiJLMjorQFNWKHNwTj1dTGpjczNdbDIjTkJ0JCdQRDo1O1UuI0hyY25yIVgjVURdNDZPJCUvKlAzU3A8NzspPG4hSS9ENF0/WFdON246Zyw/LkA5UCVpTFs6KkJrTlQvNGxDQC9acSFxaSxDZT5kZGtkNExsbmUxJl9RTV9mPjtQbDdWRTFNWldKbXInamxMcW1MUSo0X2lOaHItYmNKX1xjLldIIitwNkxPMSsxJjcnVzZaMjotUUZZNTksJ1M3PnE+QUhKLDdlYE8uKSJeSCtPPiNYPitLX0VPTjA+XWAqVHVfYUhOLlVEVnIhZWFNWE8lZUZDV09TMGNcVGE8UjdIZS9kXkhiVW5OMmZcJk9HSUdWb3QsMDhmODA8K3NSSzJtXkZ+PmVuZHN0cmVhbQplbmRvYmoKOCAwIG9iago8PAovQ29udGVudHMgMTkgMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXSAvWE9iamVjdCA8PAovRm9ybVhvYi4yYTg4YWM2NzEyYzljYTcyN2U0YWQ4ZjI1YmNiZGFlMiA3IDAgUiAvRm9ybVhvYi42YmU0MGVlYWFmMGQ4MWQxODc1MGRjMGE3ZjlkYzIxMyA1IDAgUiAvRm9ybVhvYi5lNWMzMTQ3ZDE4Zjk4NjdhNGYxMDhkM2E1NWEyNDhiNCA2IDAgUgo+Pgo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKOSAwIG9iago8PAovQSA8PAovUyAvVVJJIC9UeXBlIC9BY3Rpb24gL1VSSSAoaHR0cHM6Ly9leGFtcGxlLmNvbS9pbnZvaWNlcy9JTlYtMjA0OCkKPj4gL0JvcmRlciBbIDAgMCAwIF0gL1JlY3QgWyA0NSA1NzUgMzAwIDU5MCBdIC9TdWJ0eXBlIC9MaW5rIC9UeXBlIC9Bbm5vdAo+PgplbmRvYmoKMTAgMCBvYmoKPDwKL0MgWyAuODMgLjg5IC45NSBdIC9Db250ZW50cyAoVG90YWwgaGlnaGxpZ2h0ZWQgZm9yIHJldmlldykgL1F1YWRQb2ludHMgWyA0MCA1NzkgNTQwIDU3OSA0MCA1NTUgNTQwIDU1NSBdIC9SZWN0IFsgNDAgNTU1IDU0MCA1NzkgXSAvU3VidHlwZSAvSGlnaGxpZ2h0IC9UeXBlIC9Bbm5vdAo+PgplbmRvYmoKMTEgMCBvYmoKPDwKL0MgWyAwIDAgMCBdIC9Db250ZW50cyAoVmVyaWZ5IHRoZSBoaWdobGlnaHRlZCB0b3RhbCkgL1F1YWRQb2ludHMgWyA1MjAgNTI1IDU0MCA1MjUgNTIwIDU0NSA1NDAgNTQ1IF0gL1JlY3QgWyA1MjAgNTI1IDU0MCA1NDUgXSAvU3VidHlwZSAvVGV4dCAvVHlwZSAvQW5ub3QKPj4KZW5kb2JqCjEyIDAgb2JqCjw8Ci9Bbm5vdHMgWyA5IDAgUiAxMCAwIFIgMTEgMCBSIF0gL0NvbnRlbnRzIDIwIDAgUiAvTWVkaWFCb3ggWyAwIDAgNjEyIDc5MiBdIC9QYXJlbnQgMTcgMCBSIC9SZXNvdXJjZXMgPDwKL0ZvbnQgMSAwIFIgL1Byb2NTZXQgWyAvUERGIC9UZXh0IC9JbWFnZUIgL0ltYWdlQyAvSW1hZ2VJIF0KPj4gL1JvdGF0ZSAwIAogIC9UcmFucyA8PAoKPj4gL1R5cGUgL1BhZ2UKPj4KZW5kb2JqCjEzIDAgb2JqCjw8Ci9Db250ZW50cyAyMSAwIFIgL01lZGlhQm94IFsgMCAwIDYxMiA3OTIgXSAvUGFyZW50IDE3IDAgUiAvUmVzb3VyY2VzIDw8Ci9Gb250IDEgMCBSIC9Qcm9jU2V0IFsgL1BERiAvVGV4dCAvSW1hZ2VCIC9JbWFnZUMgL0ltYWdlSSBdCj4+IC9Sb3RhdGUgMCAvVHJhbnMgPDwKCj4+IAogIC9UeXBlIC9QYWdlCj4+CmVuZG9iagoxNCAwIG9iago8PAovQ29udGVudHMgMjIgMCBSIC9NZWRpYUJveCBbIDAgMCA2MTIgNzkyIF0gL1BhcmVudCAxNyAwIFIgL1Jlc291cmNlcyA8PAovRm9udCAxIDAgUiAvUHJvY1NldCBbIC9QREYgL1RleHQgL0ltYWdlQiAvSW1hZ2VDIC9JbWFnZUkgXQo+PiAvUm90YXRlIDAgL1RyYW5zIDw8Cgo+PiAKICAvVHlwZSAvUGFnZQo+PgplbmRvYmoKMTUgMCBvYmoKPDwKL1BhZ2VNb2RlIC9Vc2VOb25lIC9QYWdlcyAxNyAwIFIgL1R5cGUgL0NhdGFsb2cKPj4KZW5kb2JqCjE2IDAgb2JqCjw8Ci9BdXRob3IgKExpdGVMTE0gT0NSIGZpeHR1cmUgZ2VuZXJhdG9yKSAvQ3JlYXRpb25EYXRlIChEOjIwMDAwMTAxMDAwMDAwKzAwJzAwJykgL0NyZWF0b3IgKGFub255bW91cykgL0tleXdvcmRzIChPQ1IsIGludm9pY2UsIHRhYmxlLCBmaWd1cmUsIGFubm90YXRpb24pIC9Nb2REYXRlIChEOjIwMDAwMTAxMDAwMDAwKzAwJzAwJykgL1Byb2R1Y2VyIChSZXBvcnRMYWIgUERGIExpYnJhcnkgLSBcKG9wZW5zb3VyY2VcKSkgCiAgL1N1YmplY3QgKFNlbWFudGljIE9DUiBjb3ZlcmFnZSBmb3IgdGFibGVzLCBmaWd1cmVzLCBhbm5vdGF0aW9ucywgYW5kIG1ldGFkYXRhKSAvVGl0bGUgKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgL1RyYXBwZWQgL0ZhbHNlCj4+CmVuZG9iagoxNyAwIG9iago8PAovQ291bnQgNSAvS2lkcyBbIDQgMCBSIDggMCBSIDEyIDAgUiAxMyAwIFIgMTQgMCBSIF0gL1R5cGUgL1BhZ2VzCj4+CmVuZG9iagoxOCAwIG9iago8PAovTGVuZ3RoIDIyODAKPj4Kc3RyZWFtCjEgMCAwIDEgMCAwIGNtICBCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVAowIDAgMCByZwpCVCAvRjEgMTEgVGYgMTMuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc3MCBUbSAoUXVhcnRlcmx5IE9wZXJhdGlvbnMgUmVwb3J0KSBUaiBUKiBFVApCVCAvRjIgMTYgVGYgMTkuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc0NSBUbSAoSW52b2ljZSBTdW1tYXJ5IGFuZCBMaW5lIEl0ZW1zKSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgMSBvZiA1KSBUaiBUKiBFVApuIDQ1IDYyNSBtIDQ1IDczMCBsIFMKbiAyNDUgNjI1IG0gMjQ1IDczMCBsIFMKbiA0MDUgNjI1IG0gNDA1IDczMCBsIFMKbiA1NjUgNjI1IG0gNTY1IDczMCBsIFMKbiA0NSA3MzAgbSA1NjUgNzMwIGwgUwpuIDQ1IDY5NSBtIDU2NSA2OTUgbCBTCm4gNDUgNjYwIG0gNTY1IDY2MCBsIFMKbiA0NSA2MjUgbSA1NjUgNjI1IGwgUwpCVCAxIDAgMCAxIDU1IDcwNyBUbSAoSXRlbSkgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNzA3IFRtIChRdWFudGl0eSkgVGogVCogRVQKQlQgMSAwIDAgMSA0MTUgNzA3IFRtIChBbW91bnQpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNTUgNjcyIFRtIChEb2N1bWVudCBhbmFseXNpcykgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNjcyIFRtICgyKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA2NzIgVG0gKDEyMC4wMCkgVGogVCogRVQKQlQgMSAwIDAgMSA1NSA2MzcgVG0gKE9DUiB2ZXJpZmljYXRpb24pIFRqIFQqIEVUCkJUIDEgMCAwIDEgMjU1IDYzNyBUbSAoMSkgVGogVCogRVQKQlQgMSAwIDAgMSA0MTUgNjM3IFRtICg4MC4wMCkgVGogVCogRVQKbiA0NSA0OTUgbSA0NSA2MDAgbCBTCm4gMjQ1IDQ5NSBtIDI0NSA2MDAgbCBTCm4gNDA1IDQ5NSBtIDQwNSA2MDAgbCBTCm4gNTY1IDQ5NSBtIDU2NSA2MDAgbCBTCm4gNDUgNjAwIG0gNTY1IDYwMCBsIFMKbiA0NSA1NjUgbSA1NjUgNTY1IGwgUwpuIDQ1IDUzMCBtIDU2NSA1MzAgbCBTCm4gNDUgNDk1IG0gNTY1IDQ5NSBsIFMKQlQgMSAwIDAgMSA1NSA1NzcgVG0gKEl0ZW0gY29udGludWVkKSBUaiBUKiBFVApCVCAxIDAgMCAxIDI1NSA1NzcgVG0gKFF1YW50aXR5KSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA1NzcgVG0gKEFtb3VudCkgVGogVCogRVQKQlQgMSAwIDAgMSA1NSA1NDIgVG0gKEZpeHR1cmUgdmFsaWRhdGlvbikgVGogVCogRVQKQlQgMSAwIDAgMSAyNTUgNTQyIFRtICgzKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQxNSA1NDIgVG0gKDQ1LjAwKSBUaiBUKiBFVApCVCAxIDAgMCAxIDU1IDUwNyBUbSAoUHJvdmlkZXIgcmV2aWV3KSBUaiBUKiBFVApCVCAxIDAgMCAxIDI1NSA1MDcgVG0gKDEpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDE1IDUwNyBUbSAoMjUuMDApIFRqIFQqIEVUCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiAxLjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiAxLjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiAxLjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiAxLjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiAxLjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiAxLjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiAxLjc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiAxLjg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKMTkgMCBvYmoKPDwKL0xlbmd0aCAxNzIzCj4+CnN0cmVhbQoxIDAgMCAxIDAgMCBjbSAgQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKMCAwIDAgcmcKQlQgL0YxIDExIFRmIDEzLjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NzAgVG0gKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgVGogVCogRVQKQlQgL0YyIDE2IFRmIDE5LjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NDUgVG0gKFJldmVudWUgQ2hhcnQgYW5kIEZvcm11bGEgUmV2aWV3KSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgMiBvZiA1KSBUaiBUKiBFVAouMiAuNDUwOTggLjg1MDk4IHJnCm4gNzAgNjEwIDY1IDcwIHJlIGYqCm4gMTcwIDYxMCA2NSAxMTUgcmUgZioKbiAyNzAgNjEwIDY1IDkwIHJlIGYqCm4gMzcwIDYxMCA2NSAxMzAgcmUgZioKMCAwIDAgcmcKQlQgMSAwIDAgMSA5MCA1OTAgVG0gKFExKSBUaiBUKiBFVApCVCAxIDAgMCAxIDE5MCA1OTAgVG0gKFEyKSBUaiBUKiBFVApCVCAxIDAgMCAxIDI5MCA1OTAgVG0gKFEzKSBUaiBUKiBFVApCVCAxIDAgMCAxIDM5MCA1OTAgVG0gKFE0KSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDU1MCBUbSAoRm9ybXVsYTogZ3Jvc3MgbWFyZ2luID0gXChyZXZlbnVlIC0gY29zdFwpIC8gcmV2ZW51ZSkgVGogVCogRVQKcQo3MCAwIDAgNzAgNDU1IDY1NSBjbQovRm9ybVhvYi42YmU0MGVlYWFmMGQ4MWQxODc1MGRjMGE3ZjlkYzIxMyBEbwpRCnEKNzAgMCAwIDYyIDQ1NSA1NjUgY20KL0Zvcm1Yb2IuZTVjMzE0N2QxOGY5ODY3YTRmMTA4ZDNhNTVhMjQ4YjQgRG8KUQpxCjQ1IDAgMCA0NSA0NTUgNTAwIGNtCi9Gb3JtWG9iLjJhODhhYzY3MTJjOWNhNzI3ZTRhZDhmMjViY2JkYWUyIERvClEKQlQgL0YxIDEwIFRmIDEyIFRMIEVUCkJUIDEgMCAwIDEgNDUgNDcyIFRtIChTZWN0aW9uIDIuMTogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNDQ0IFRtIChTZWN0aW9uIDIuMjogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNDE2IFRtIChTZWN0aW9uIDIuMzogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzg4IFRtIChTZWN0aW9uIDIuNDogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzYwIFRtIChTZWN0aW9uIDIuNTogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzMyIFRtIChTZWN0aW9uIDIuNjogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMzA0IFRtIChTZWN0aW9uIDIuNzogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgMjc2IFRtIChTZWN0aW9uIDIuODogSW52b2ljZSB0b3RhbHMsIHJlZ2lvbmFsIHJldmVudWUsIGFuZCByZWNvbmNpbGlhdGlvbiBub3Rlcy4pIFRqIFQqIEVUCiAKZW5kc3RyZWFtCmVuZG9iagoyMCAwIG9iago8PAovTGVuZ3RoIDI0NjYKPj4Kc3RyZWFtCjEgMCAwIDEgMCAwIGNtICBCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVAowIDAgMCByZwpCVCAvRjEgMTEgVGYgMTMuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc3MCBUbSAoUXVhcnRlcmx5IE9wZXJhdGlvbnMgUmVwb3J0KSBUaiBUKiBFVApCVCAvRjIgMTYgVGYgMTkuMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDc0NSBUbSAoS2V5IFZhbHVlcywgTGluaywgSGlnaGxpZ2h0LCBhbmQgQ29tbWVudCkgVGogVCogRVQKQlQgL0YxIDkgVGYgMTAuOCBUTCBFVApCVCAxIDAgMCAxIDQ1IDMwIFRtIChDb25maWRlbnRpYWwgfCBQYWdlIDMgb2YgNSkgVGogVCogRVQKQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKQlQgMSAwIDAgMSA0NSA3MDAgVG0gKEludm9pY2UgTnVtYmVyOiBJTlYtMjA0OCkgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA2NzUgVG0gKFB1cmNoYXNlIE9yZGVyOiBQTy00MDk2KSBUaiBUKiBFVAouOTQ5MDIgLjkwMTk2MSAuMzUyOTQxIHJnCm4gNDAgNTU1IDUwMCAyNCByZSBmKgowIDAgMCByZwpCVCAxIDAgMCAxIDQ1IDU2MCBUbSAoSGlnaGxpZ2h0ZWQgdG90YWwgcmVxdWlyaW5nIHJldmlldykgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA1MzAgVG0gKFJldmlld2VyIGNvbW1lbnQ6IHZlcmlmeSB0aGUgaGlnaGxpZ2h0ZWQgdG90YWwgYmVmb3JlIGFwcHJvdmFsKSBUaiBUKiBFVAoxIDAgMCByZwpCVCAxIDAgMCAxIDQ1IDQ5NSBUbSAoUmV2aXNlZCB0b3RhbDogMjQ1LjAwKSBUaiBUKiBFVApuIDQ1IDUwMSBtIDE1MCA1MDEgbCBTCjAgMCAwIHJnCkJUIDEgMCAwIDEgNDUgNTc1IFRtIChodHRwczovL2V4YW1wbGUuY29tL2ludm9pY2VzL0lOVi0yMDQ4KSBUaiBUKiBFVApxCjEgMCAwIDEgOTAgMTMwIGNtCm4gMTggMCAyLjQgNzAgcmUgZioKbiAyMS42IDAgMS4yIDcwIHJlIGYqCm4gMjUuMiAwIDMuNiA3MCByZSBmKgpuIDMxLjIgMCAzLjYgNzAgcmUgZioKbiAzOC40IDAgMi40IDcwIHJlIGYqCm4gNDIgMCAxLjIgNzAgcmUgZioKbiA0NC40IDAgMi40IDcwIHJlIGYqCm4gNDkuMiAwIDIuNCA3MCByZSBmKgpuIDUyLjggMCAyLjQgNzAgcmUgZioKbiA1Ny42IDAgMy42IDcwIHJlIGYqCm4gNjIuNCAwIDIuNCA3MCByZSBmKgpuIDY2IDAgMy42IDcwIHJlIGYqCm4gNzAuOCAwIDIuNCA3MCByZSBmKgpuIDc2LjggMCAxLjIgNzAgcmUgZioKbiA4MS42IDAgMS4yIDcwIHJlIGYqCm4gODQgMCAzLjYgNzAgcmUgZioKbiA4OC44IDAgMi40IDcwIHJlIGYqCm4gOTIuNCAwIDMuNiA3MCByZSBmKgpuIDk3LjIgMCAxLjIgNzAgcmUgZioKbiA5OS42IDAgMy42IDcwIHJlIGYqCm4gMTA0LjQgMCAyLjQgNzAgcmUgZioKbiAxMTAuNCAwIDEuMiA3MCByZSBmKgpuIDExMi44IDAgNC44IDcwIHJlIGYqCm4gMTE4LjggMCAzLjYgNzAgcmUgZioKbiAxMjMuNiAwIDMuNiA3MCByZSBmKgpuIDEyOC40IDAgMi40IDcwIHJlIGYqCm4gMTMyIDAgMy42IDcwIHJlIGYqCm4gMTM2LjggMCAyLjQgNzAgcmUgZioKbiAxNDEuNiAwIDEuMiA3MCByZSBmKgpuIDE0NS4yIDAgMy42IDcwIHJlIGYqCm4gMTUwIDAgMi40IDcwIHJlIGYqCm4gMTU2IDAgMy42IDcwIHJlIGYqCm4gMTYwLjggMCAxLjIgNzAgcmUgZioKbiAxNjMuMiAwIDIuNCA3MCByZSBmKgpRCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiAzLjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiAzLjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiAzLjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiAzLjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiAzLjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiAzLjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiAzLjc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiAzLjg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKMjEgMCBvYmoKPDwKL0xlbmd0aCAxNDk0Cj4+CnN0cmVhbQoxIDAgMCAxIDAgMCBjbSAgQlQgL0YxIDEyIFRmIDE0LjQgVEwgRVQKMCAwIDAgcmcKQlQgL0YxIDExIFRmIDEzLjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NzAgVG0gKFF1YXJ0ZXJseSBPcGVyYXRpb25zIFJlcG9ydCkgVGogVCogRVQKQlQgL0YyIDE2IFRmIDE5LjIgVEwgRVQKQlQgMSAwIDAgMSA0NSA3NDUgVG0gKEFwcHJvdmFsIFNpZ25hdHVyZSBhbmQgV2F0ZXJtYXJrKSBUaiBUKiBFVApCVCAvRjEgOSBUZiAxMC44IFRMIEVUCkJUIDEgMCAwIDEgNDUgMzAgVG0gKENvbmZpZGVudGlhbCB8IFBhZ2UgNCBvZiA1KSBUaiBUKiBFVApxCi44Mjc0NTEgLjgyNzQ1MSAuODI3NDUxIHJnCkJUIC9GMiA1NCBUZiA2NC44IFRMIEVUCi45MDYzMDggLjQyMjYxOCAtMC40MjI2MTggLjkwNjMwOCAxMTAgMzkwIGNtCkJUIDEgMCAwIDEgMCAwIFRtIChEUkFGVCkgVGogVCogRVQKUQowIDAgMCByZwpCVCAvRjEgMTIgVGYgMTQuNCBUTCBFVApCVCAxIDAgMCAxIDQ1IDYzNSBUbSAoQXBwcm92ZWQgYnk6IEpvcmRhbiBMZWUpIFRqIFQqIEVUCm4gNDUgNjEwIG0gMzEwIDYxMCBsIFMKbiA1NSA1OTUgbSA3NSA2MjUgMTEyIDYwMiAxNTUgNjAwIGMgUwpCVCAxIDAgMCAxIDQ1IDU4MCBUbSAoU2lnbmF0dXJlKSBUaiBUKiBFVApCVCAvRjEgMTAgVGYgMTIgVEwgRVQKQlQgMSAwIDAgMSA0NSA0NzIgVG0gKFNlY3Rpb24gNC4xOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA0NDQgVG0gKFNlY3Rpb24gNC4yOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSA0MTYgVG0gKFNlY3Rpb24gNC4zOiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzODggVG0gKFNlY3Rpb24gNC40OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzNjAgVG0gKFNlY3Rpb24gNC41OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzMzIgVG0gKFNlY3Rpb24gNC42OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAzMDQgVG0gKFNlY3Rpb24gNC43OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKQlQgMSAwIDAgMSA0NSAyNzYgVG0gKFNlY3Rpb24gNC44OiBJbnZvaWNlIHRvdGFscywgcmVnaW9uYWwgcmV2ZW51ZSwgYW5kIHJlY29uY2lsaWF0aW9uIG5vdGVzLikgVGogVCogRVQKIAplbmRzdHJlYW0KZW5kb2JqCjIyIDAgb2JqCjw8Ci9MZW5ndGggMTMyNwo+PgpzdHJlYW0KMSAwIDAgMSAwIDAgY20gIEJUIC9GMSAxMiBUZiAxNC40IFRMIEVUCjAgMCAwIHJnCkJUIC9GMSAxMSBUZiAxMy4yIFRMIEVUCkJUIDEgMCAwIDEgNDUgNzcwIFRtIChRdWFydGVybHkgT3BlcmF0aW9ucyBSZXBvcnQpIFRqIFQqIEVUCkJUIC9GMiAxNiBUZiAxOS4yIFRMIEVUCkJUIDEgMCAwIDEgNDUgNzQ1IFRtIChBcHBlbmRpeCB3aXRoIFNlY3Rpb24gQm91bmRhcmllcykgVGogVCogRVQKQlQgL0YxIDkgVGYgMTAuOCBUTCBFVApCVCAxIDAgMCAxIDQ1IDMwIFRtIChDb25maWRlbnRpYWwgfCBQYWdlIDUgb2YgNSkgVGogVCogRVQKQlQgL0YyIDE0IFRmIDE2LjggVEwgRVQKQlQgMSAwIDAgMSA0NSA3MDAgVG0gKDEuIFNjb3BlKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDY1MCBUbSAoMi4gRmluZGluZ3MpIFRqIFQqIEVUCkJUIDEgMCAwIDEgNDUgNjAwIFRtICgzLiBSZWNvbW1lbmRhdGlvbnMpIFRqIFQqIEVUCkJUIC9GMSAxMCBUZiAxMiBUTCBFVApCVCAxIDAgMCAxIDQ1IDQ3MiBUbSAoU2VjdGlvbiA1LjE6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQ0NCBUbSAoU2VjdGlvbiA1LjI6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDQxNiBUbSAoU2VjdGlvbiA1LjM6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM4OCBUbSAoU2VjdGlvbiA1LjQ6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDM2MCBUbSAoU2VjdGlvbiA1LjU6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMzMiBUbSAoU2VjdGlvbiA1LjY6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDMwNCBUbSAoU2VjdGlvbiA1Ljc6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVApCVCAxIDAgMCAxIDQ1IDI3NiBUbSAoU2VjdGlvbiA1Ljg6IEludm9pY2UgdG90YWxzLCByZWdpb25hbCByZXZlbnVlLCBhbmQgcmVjb25jaWxpYXRpb24gbm90ZXMuKSBUaiBUKiBFVAogCmVuZHN0cmVhbQplbmRvYmoKeHJlZgowIDIzCjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDA2MSAwMDAwMCBuIAowMDAwMDAwMTAyIDAwMDAwIG4gCjAwMDAwMDAyMDkgMDAwMDAgbiAKMDAwMDAwMDMyMSAwMDAwMCBuIAowMDAwMDAwNTE2IDAwMDAwIG4gCjAwMDAwMDYzMjIgMDAwMDAgbiAKMDAwMDAxMjgyNCAwMDAwMCBuIAowMDAwMDE0MTQyIDAwMDAwIG4gCjAwMDAwMTQ0OTYgMDAwMDAgbiAKMDAwMDAxNDY2NCAwMDAwMCBuIAowMDAwMDE0ODUwIDAwMDAwIG4gCjAwMDAwMTUwMjggMDAwMDAgbiAKMDAwMDAxNTI1NiAwMDAwMCBuIAowMDAwMDE1NDUyIDAwMDAwIG4gCjAwMDAwMTU2NDggMDAwMDAgbiAKMDAwMDAxNTcxOCAwMDAwMCBuIAowMDAwMDE2MTE1IDAwMDAwIG4gCjAwMDAwMTYyMDIgMDAwMDAgbiAKMDAwMDAxODUzNCAwMDAwMCBuIAowMDAwMDIwMzA5IDAwMDAwIG4gCjAwMDAwMjI4MjcgMDAwMDAgbiAKMDAwMDAyNDM3MyAwMDAwMCBuIAp0cmFpbGVyCjw8Ci9JRCAKWzwyZjUwODkzYTFlYWZmMTExOWMwNDcwZmM0YzU3ZTI0Nz48MmY1MDg5M2ExZWFmZjExMTljMDQ3MGZjNGM1N2UyNDc+XQolIFJlcG9ydExhYiBnZW5lcmF0ZWQgUERGIGRvY3VtZW50IC0tIGRpZ2VzdCAob3BlbnNvdXJjZSkKCi9JbmZvIDE2IDAgUgovUm9vdCAxNSAwIFIKL1NpemUgMjMKPj4Kc3RhcnR4cmVmCjI1NzUyCiUlRU9GCg== + type: document_url + model: mistral/invalid-ocr-model-for-parity + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/1f38e1eab857c22107df833aa63b119bced6537f45cf08b27cbec08bb8f45951.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/1f38e1eab857c22107df833aa63b119bced6537f45cf08b27cbec08bb8f45951.yaml new file mode 100644 index 00000000000..8200c885e60 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/1f38e1eab857c22107df833aa63b119bced6537f45cf08b27cbec08bb8f45951.yaml @@ -0,0 +1,69 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"pages":[0]}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d35d1fbbebe5-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:14 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-5685-747f-b76d-dab242ea7512 + x-envoy-upstream-service-time: + - '179' + x-kong-proxy-latency: + - '12' + x-kong-request-id: + - 01a05e89-5685-747f-b76d-dab242ea7512 + x-kong-upstream-latency: + - '180' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '58' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:14.385374+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + pages: + - 0 + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/38dbc05d8508355879de894ecc0f46da887570de720a142e9df297bdd280608d.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/38dbc05d8508355879de894ecc0f46da887570de720a142e9df297bdd280608d.yaml new file mode 100644 index 00000000000..a9fd2d0fbec --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/38dbc05d8508355879de894ecc0f46da887570de720a142e9df297bdd280608d.yaml @@ -0,0 +1,85 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"document_annotation_prompt":"Extract + the visible title"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\": + \"invoice 123\"}","usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d376581f74f9-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:18 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-664a-740d-a112-0a1183c302b3 + x-envoy-upstream-service-time: + - '402' + x-kong-proxy-latency: + - '20' + x-kong-request-id: + - 01a05e89-664a-740d-a112-0a1183c302b3 + x-kong-upstream-latency: + - '403' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '52' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:18.915814+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + document_annotation_format: + json_schema: + description: Extract the visible document fields + name: document_title + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + document_annotation_prompt: Extract the visible title + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/3d5f42192461789f547b42c8f1b625aa688e6cbd69e01f94d1f3205ef41ab830.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/3d5f42192461789f547b42c8f1b625aa688e6cbd69e01f94d1f3205ef41ab830.yaml new file mode 100644 index 00000000000..a471b8f7bae --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/3d5f42192461789f547b42c8f1b625aa688e6cbd69e01f94d1f3205ef41ab830.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"extract_header":true}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d37c9944d8a7-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:19 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-6a3a-76c7-85b8-d40a7156155e + x-envoy-upstream-service-time: + - '230' + x-kong-proxy-latency: + - '16' + x-kong-request-id: + - 01a05e89-6a3a-76c7-85b8-d40a7156155e + x-kong-upstream-latency: + - '230' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '51' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:19.420285+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + extract_header: true + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/54ce0991a21ea668e2373d2b05801606b7039583f55a0f85bfe38c7c15712adf.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/54ce0991a21ea668e2373d2b05801606b7039583f55a0f85bfe38c7c15712adf.yaml new file mode 100644 index 00000000000..90a4abd5574 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/54ce0991a21ea668e2373d2b05801606b7039583f55a0f85bfe38c7c15712adf.yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d356fb3698ce-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:13 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-52b4-7e7b-983e-5bc4e5469b39 + x-envoy-upstream-service-time: + - '554' + x-kong-proxy-latency: + - '13' + x-kong-request-id: + - 01a05e89-52b4-7e7b-983e-5bc4e5469b39 + x-kong-upstream-latency: + - '557' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '59' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:13.880852+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/7f7dc004625cb90e3f4bd2f473fda532390140ade4e10b5bb07570ca7b372499.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/7f7dc004625cb90e3f4bd2f473fda532390140ade4e10b5bb07570ca7b372499.yaml new file mode 100644 index 00000000000..748948e5e8c --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/7f7dc004625cb90e3f4bd2f473fda532390140ade4e10b5bb07570ca7b372499.yaml @@ -0,0 +1,82 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"bbox_annotation_format":{"type":"json_schema","json_schema":{"name":"bounding_boxes","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d369be838fc5-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:16 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-5e6e-70ac-890d-c0110a5e81af + x-envoy-upstream-service-time: + - '210' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 01a05e89-5e6e-70ac-890d-c0110a5e81af + x-kong-upstream-latency: + - '212' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '54' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:16.402593+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + bbox_annotation_format: + json_schema: + description: Extract the visible document fields + name: bounding_boxes + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/99d9d5ac0449213a562a36163ee598ef9e6df1813bee21137e88bcc14ce7a473.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/99d9d5ac0449213a562a36163ee598ef9e6df1813bee21137e88bcc14ce7a473.yaml new file mode 100644 index 00000000000..80c9a75578a --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/99d9d5ac0449213a562a36163ee598ef9e6df1813bee21137e88bcc14ce7a473.yaml @@ -0,0 +1,92 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"document_url","document_url":"data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg=="},"pages":[0],"include_image_base64":true,"image_limit":1,"image_min_size":300,"bbox_annotation_format":{"type":"json_schema","json_schema":{"name":"bounding_boxes","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"extract_header":true,"extract_footer":false,"table_format":"markdown","confidence_scores_granularity":"page","include_blocks":false,"id":"case-1"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"Test PDF File","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":93,"height":1023,"width":791},"confidence_scores":{"word_confidence_scores":[],"average_page_confidence_score":0.9376229744322936,"minimum_page_confidence_score":0.22590550796036835},"blocks":null}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":589}}' + headers: + CF-RAY: + - a346d39c1fe5cf12-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:24 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7deb-72b4-976d-4c244b056782 + x-envoy-upstream-service-time: + - '373' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 01a05e89-7deb-72b4-976d-4c244b056782 + x-kong-upstream-latency: + - '373' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '45' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:24.951956+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + bbox_annotation_format: + json_schema: + description: Extract the visible document fields + name: bounding_boxes + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + confidence_scores_granularity: page + contract: mistral + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + extract_footer: false + extract_header: true + id: case-1 + image_limit: 1 + image_min_size: 300 + include_blocks: false + include_image_base64: true + model: mistral/mistral-ocr-latest + pages: + - 0 + table_format: markdown + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/9b9fb2f3a7eb8ca1e2d128ea96b358dc7c9b0eafc64e7e2516d46e290fc93092.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/9b9fb2f3a7eb8ca1e2d128ea96b358dc7c9b0eafc64e7e2516d46e290fc93092.yaml new file mode 100644 index 00000000000..847bac3f431 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/9b9fb2f3a7eb8ca1e2d128ea96b358dc7c9b0eafc64e7e2516d46e290fc93092.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\": + \"Invoice_123\"}","usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d36cd82f15ba-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:17 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-6060-71cb-9e35-66ab873b99f3 + x-envoy-upstream-service-time: + - '888' + x-kong-proxy-latency: + - '12' + x-kong-request-id: + - 01a05e89-6060-71cb-9e35-66ab873b99f3 + x-kong-upstream-latency: + - '889' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '53' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:17.908846+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + document_annotation_format: + json_schema: + description: Extract the visible document fields + name: document_title + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/a8693352d9ce0d7d120b2351d2257cd562048fc6a993cba093fe86a8937d104b.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/a8693352d9ce0d7d120b2351d2257cd562048fc6a993cba093fe86a8937d104b.yaml new file mode 100644 index 00000000000..3b5f8bb85e1 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/a8693352d9ce0d7d120b2351d2257cd562048fc6a993cba093fe86a8937d104b.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"include_image_base64":true}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d360593c138a-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:14 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-588e-7e32-a54b-f816bc6c1dc4 + x-envoy-upstream-service-time: + - '242' + x-kong-proxy-latency: + - '16' + x-kong-request-id: + - 01a05e89-588e-7e32-a54b-f816bc6c1dc4 + x-kong-upstream-latency: + - '242' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '57' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:14.889667+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + include_image_base64: true + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/b19fdc140375a4eadfd6eb4e4cbe88e27651580bac1e145ad352728b1a0acd31.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/b19fdc140375a4eadfd6eb4e4cbe88e27651580bac1e145ad352728b1a0acd31.yaml new file mode 100644 index 00000000000..acabdaf2b5e --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/b19fdc140375a4eadfd6eb4e4cbe88e27651580bac1e145ad352728b1a0acd31.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"image_min_size":300}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d36688d203c2-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:15 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-5c60-769b-abf3-44dcefdf5282 + x-envoy-upstream-service-time: + - '272' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 01a05e89-5c60-769b-abf3-44dcefdf5282 + x-kong-upstream-latency: + - '273' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '55' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:15.898087+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + image_min_size: 300 + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/bdb616d21dece89481debccb9358047b28006b5cbd4486d54c6fe89070e7a671.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/bdb616d21dece89481debccb9358047b28006b5cbd4486d54c6fe89070e7a671.yaml new file mode 100644 index 00000000000..ed060cd05ca --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/bdb616d21dece89481debccb9358047b28006b5cbd4486d54c6fe89070e7a671.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"id":"case-1"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d398da783ad4-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:23 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7bdc-74c5-b8eb-15fbd25b6cfd + x-envoy-upstream-service-time: + - '182' + x-kong-proxy-latency: + - '22' + x-kong-request-id: + - 01a05e89-7bdc-74c5-b8eb-15fbd25b6cfd + x-kong-upstream-latency: + - '183' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '46' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:23.946966+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + id: case-1 + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c3066e2f2f964725b71ae891821d77447359bba8c29d6b31e650255298271d47.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c3066e2f2f964725b71ae891821d77447359bba8c29d6b31e650255298271d47.yaml new file mode 100644 index 00000000000..22d01018046 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c3066e2f2f964725b71ae891821d77447359bba8c29d6b31e650255298271d47.yaml @@ -0,0 +1,67 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"include_blocks":false}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":null}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d392a8432af7-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:22 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7802-768c-bf57-9021a209ab48 + x-envoy-upstream-service-time: + - '213' + x-kong-proxy-latency: + - '17' + x-kong-request-id: + - 01a05e89-7802-768c-bf57-9021a209ab48 + x-kong-upstream-latency: + - '214' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '47' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:23.442341+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + include_blocks: false + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c57601c90e9b289e506cdc818523bba41a29b707d0cfafd74986e6ad8c2ed900.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c57601c90e9b289e506cdc818523bba41a29b707d0cfafd74986e6ad8c2ed900.yaml new file mode 100644 index 00000000000..9d1a1e55835 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c57601c90e9b289e506cdc818523bba41a29b707d0cfafd74986e6ad8c2ed900.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"confidence_scores_granularity":"page"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":{"word_confidence_scores":[],"average_page_confidence_score":0.90845564554897,"minimum_page_confidence_score":0.16168208839823475},"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d38c59681749-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:22 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7411-7d2c-9e43-01320e4cad1c + x-envoy-upstream-service-time: + - '329' + x-kong-proxy-latency: + - '14' + x-kong-request-id: + - 01a05e89-7411-7d2c-9e43-01320e4cad1c + x-kong-upstream-latency: + - '330' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '48' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:22.437385+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + confidence_scores_granularity: page + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c5d33cb2b20fb76c9543ff57dc9b3b7dc37c5bf5dfd47e873c4cc695d6d7461f.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c5d33cb2b20fb76c9543ff57dc9b3b7dc37c5bf5dfd47e873c4cc695d6d7461f.yaml new file mode 100644 index 00000000000..7afaa689756 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c5d33cb2b20fb76c9543ff57dc9b3b7dc37c5bf5dfd47e873c4cc695d6d7461f.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"extract_footer":true}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d37fde0f1703-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:20 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-6c42-7ea1-b08b-07cae50734e0 + x-envoy-upstream-service-time: + - '523' + x-kong-proxy-latency: + - '13' + x-kong-request-id: + - 01a05e89-6c42-7ea1-b08b-07cae50734e0 + x-kong-upstream-latency: + - '525' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '50' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:20.425444+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + extract_footer: true + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c6f0f158205be7fc00bd962e341689c3583b09694a7ac086e4707be49b35eb06.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c6f0f158205be7fc00bd962e341689c3583b09694a7ac086e4707be49b35eb06.yaml new file mode 100644 index 00000000000..09c39780583 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/c6f0f158205be7fc00bd962e341689c3583b09694a7ac086e4707be49b35eb06.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"document_url","document_url":"data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg=="},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"Test PDF File","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":93,"height":1023,"width":791},"confidence_scores":null,"blocks":[{"top_left_x":126,"top_left_y":104,"bottom_right_x":229,"bottom_right_y":127,"content":"Test + PDF File","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\": + \"Test_PDF_File\"}","usage_info":{"pages_processed":1,"doc_size_bytes":589}}' + headers: + CF-RAY: + - a346d3a25d0cccb8-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:25 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-81d5-79f8-914d-e3863af2d24b + x-envoy-upstream-service-time: + - '448' + x-kong-proxy-latency: + - '15' + x-kong-request-id: + - 01a05e89-81d5-79f8-914d-e3863af2d24b + x-kong-upstream-latency: + - '449' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '44' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:25.959391+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + document_annotation_format: + json_schema: + description: Extract the visible document fields + name: document_title + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + model: mistral/mistral-ocr-latest + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/cea250810b49cd17251ee8c036eae74dd363ddda0d6677155afbd145d0dc3c36.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/cea250810b49cd17251ee8c036eae74dd363ddda0d6677155afbd145d0dc3c36.yaml new file mode 100644 index 00000000000..85d37a96634 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/cea250810b49cd17251ee8c036eae74dd363ddda0d6677155afbd145d0dc3c36.yaml @@ -0,0 +1,68 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"table_format":"markdown"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":null,"usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d385ffbda0f2-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:21 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-7015-7a8d-a18c-37b9ceacbe77 + x-envoy-upstream-service-time: + - '337' + x-kong-proxy-latency: + - '21' + x-kong-request-id: + - 01a05e89-7015-7a8d-a18c-37b9ceacbe77 + x-kong-upstream-latency: + - '338' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '49' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:21.430472+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + model: mistral/mistral-ocr-latest + table_format: markdown + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/d80a6ccca02922ba9094426235a6ab165f1866eb074a8c079273eef5df512985.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/d80a6ccca02922ba9094426235a6ab165f1866eb074a8c079273eef5df512985.yaml new file mode 100644 index 00000000000..5e8fe1a1cdc --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/mistral-ocr/d80a6ccca02922ba9094426235a6ab165f1866eb074a8c079273eef5df512985.yaml @@ -0,0 +1,105 @@ +interactions: +- request: + body: '{"model":"mistral-ocr-latest","document":{"type":"image_url","image_url":"https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24"},"pages":[0],"include_image_base64":false,"image_min_size":300,"bbox_annotation_format":{"type":"json_schema","json_schema":{"name":"bounding_boxes","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"document_annotation_format":{"type":"json_schema","json_schema":{"name":"document_title","description":"Extract + the visible document fields","schema":{"additionalProperties":false,"properties":{"title":{"type":"string"}},"required":["title"],"type":"object"},"strict":true}},"extract_header":true,"table_format":"markdown","include_blocks":true}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/v1/ocr + response: + body: + string: '{"pages":[{"index":0,"markdown":"invoice 123","images":[],"tables":[],"hyperlinks":[],"header":null,"footer":null,"dimensions":{"dpi":200,"height":300,"width":800},"confidence_scores":null,"blocks":[{"top_left_x":335,"top_left_y":138,"bottom_right_x":464,"bottom_right_y":162,"content":"invoice + 123","confidence_scores":null,"type":"text"}]}],"model":"mistral-ocr-latest","document_annotation":"{\"title\": + \"Invoice_123\"}","usage_info":{"pages_processed":1,"doc_size_bytes":4124}}' + headers: + CF-RAY: + - a346d3a89a85f953-SJC + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:26 GMT + Server: + - cloudflare + Strict-Transport-Security: + - max-age=15552000; includeSubDomains; preload + X-Content-Type-Options: + - nosniff + access-control-allow-origin: + - '*' + alt-svc: + - h3=":443"; ma=86400 + cf-cache-status: + - DYNAMIC + mistral-correlation-id: + - 01a05e89-85bd-7075-af53-8e341b35a63e + x-envoy-upstream-service-time: + - '432' + x-kong-proxy-latency: + - '13' + x-kong-request-id: + - 01a05e89-85bd-7075-af53-8e341b35a63e + x-kong-upstream-latency: + - '433' + x-ratelimit-limit-ocr-pages-minute: + - '60' + x-ratelimit-ocr-pages-query-cost: + - '1' + x-ratelimit-remaining-ocr-pages-minute: + - '43' + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:26.965078+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + bbox_annotation_format: + json_schema: + description: Extract the visible document fields + name: bounding_boxes + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + contract: mistral + document: + image_url: https://dummyjson.com/image/800x300/ffffff/000000?text=invoice%20123&fontSize=24 + type: image_url + document_annotation_format: + json_schema: + description: Extract the visible document fields + name: document_title + schema: + additionalProperties: false + properties: + title: + type: string + required: + - title + type: object + strict: true + type: json_schema + extract_header: true + image_min_size: 300 + include_blocks: true + include_image_base64: false + model: mistral/mistral-ocr-latest + pages: + - 0 + table_format: markdown + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/09cc2f0155ffdc3d25abff96797030105ad107ae3400532b20ad70ebbd021954.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/09cc2f0155ffdc3d25abff96797030105ad107ae3400532b20ad70ebbd021954.yaml new file mode 100644 index 00000000000..36cb1ff8ce1 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/09cc2f0155ffdc3d25abff96797030105ad107ae3400532b20ad70ebbd021954.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: "--3d54cb2e388c04dbfc172c1497aaa396\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--3d54cb2e388c04dbfc172c1497aaa396--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=3d54cb2e388c04dbfc172c1497aaa396 + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://45d3cbbc-4d77-4967-8941-935b0c4a0493.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:58 GMT + status: + code: 200 + message: '' +- request: + body: '{"document_url":"reducto://45d3cbbc-4d77-4967-8941-935b0c4a0493.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"262ed683-ecd8-40b9-a568-f7b7014854c9","duration":2.4146597385406494,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/a91f568b-6e98-4962-9924-097202093779.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195456Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=aa77b33e20fc1426307688f55147d57e8d3b2286a787c096b2da016d7d7d0c93","studio_link":"https://studio.reducto.ai/job/cddd635e-5d16-4621-bf48-031e7637dc0b","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16352969387356803,"top":0.10490237663507056,"width":0.11855640077405508,"height":0.011359045881442221,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7126715332269669},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:59 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:59.198829+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_legacy + custom_llm_provider: reducto + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: parse-legacy + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/7c4bbc65dfbc17bac7f0d179c5d7e201dfb9ac31de66094734ef38c288293625.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/7c4bbc65dfbc17bac7f0d179c5d7e201dfb9ac31de66094734ef38c288293625.yaml new file mode 100644 index 00000000000..deb47484ec8 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/7c4bbc65dfbc17bac7f0d179c5d7e201dfb9ac31de66094734ef38c288293625.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"document_url":"reducto://invalid-document-for-parity"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"error":{"code":404,"name":"NOT_FOUND","message":"Document ''The file + may have expired or been deleted. Please re-upload and try again.'' not found"},"detail":"Document + ''The file may have expired or been deleted. Please re-upload and try again.'' + not found"}' + headers: + Content-Type: + - application/json + Date: + - Wed, 02 Sep 2026 01:14:06 GMT + status: + code: 404 + message: '' +recorded_at: '2026-09-02T01:14:06.814847+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_legacy + document: + document_url: reducto://invalid-document-for-parity + type: document_url + model: reducto/parse-legacy + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/953d2a3bfea5594f9df19fc3f6c37e51b7e915b96a85e4ce8dfc41ba5f7f11df.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/953d2a3bfea5594f9df19fc3f6c37e51b7e915b96a85e4ce8dfc41ba5f7f11df.yaml new file mode 100644 index 00000000000..1e1c7c44463 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/953d2a3bfea5594f9df19fc3f6c37e51b7e915b96a85e4ce8dfc41ba5f7f11df.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: "--90e8e7e6a71b2a4b14695234a736d695\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--90e8e7e6a71b2a4b14695234a736d695--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=90e8e7e6a71b2a4b14695234a736d695 + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://b9f242fd-fdb4-4b0a-9535-f11f432c7678.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:57 GMT + status: + code: 200 + message: '' +- request: + body: '{"document_url":"reducto://b9f242fd-fdb4-4b0a-9535-f11f432c7678.pdf","options":{"enhance":{}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"25b78189-66fe-46d9-8114-14372a9d442d","duration":2.4146597385406494,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/a91f568b-6e98-4962-9924-097202093779.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195456Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=aa77b33e20fc1426307688f55147d57e8d3b2286a787c096b2da016d7d7d0c93","studio_link":"https://studio.reducto.ai/job/cddd635e-5d16-4621-bf48-031e7637dc0b","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16352969387356803,"top":0.10490237663507056,"width":0.11855640077405508,"height":0.011359045881442221,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7126715332269669},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:57 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:58.471820+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_legacy + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + enhance: {} + model: reducto/parse-legacy + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/a41d4c9547448df927b308d619d7e3b8fdbfe1fb5a9818a2fe47f02c2aad0fac.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/a41d4c9547448df927b308d619d7e3b8fdbfe1fb5a9818a2fe47f02c2aad0fac.yaml new file mode 100644 index 00000000000..a7a927cf93c --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-legacy/a41d4c9547448df927b308d619d7e3b8fdbfe1fb5a9818a2fe47f02c2aad0fac.yaml @@ -0,0 +1,69 @@ +interactions: +- request: + body: "--87aea09e72c511f78d32f6eabd194f5c\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--87aea09e72c511f78d32f6eabd194f5c--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=87aea09e72c511f78d32f6eabd194f5c + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://a91f568b-6e98-4962-9924-097202093779.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:53 GMT + status: + code: 200 + message: '' +- request: + body: '{"document_url":"reducto://a91f568b-6e98-4962-9924-097202093779.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"cddd635e-5d16-4621-bf48-031e7637dc0b","duration":2.4146597385406494,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/a91f568b-6e98-4962-9924-097202093779.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195456Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=aa77b33e20fc1426307688f55147d57e8d3b2286a787c096b2da016d7d7d0c93","studio_link":"https://studio.reducto.ai/job/cddd635e-5d16-4621-bf48-031e7637dc0b","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16352969387356803,"top":0.10490237663507056,"width":0.11855640077405508,"height":0.011359045881442221,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7126715332269669},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:57 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:57.246119+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_legacy + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-legacy + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/2148dfafc823aea933d7c9aa8877398c1c26cf1fb8f134b1326e79bd3d487c41.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/2148dfafc823aea933d7c9aa8877398c1c26cf1fb8f134b1326e79bd3d487c41.yaml new file mode 100644 index 00000000000..474e6aba8c3 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/2148dfafc823aea933d7c9aa8877398c1c26cf1fb8f134b1326e79bd3d487c41.yaml @@ -0,0 +1,72 @@ +interactions: +- request: + body: "--3c37fd2676c46ae1ff34a706baa70fad\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--3c37fd2676c46ae1ff34a706baa70fad--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=3c37fd2676c46ae1ff34a706baa70fad + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://671d6e00-6df5-493a-bd9e-8bbf3d77d3ab.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:36 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://671d6e00-6df5-493a-bd9e-8bbf3d77d3ab.pdf","retrieval":{"chunking":{"chunk_mode":"page"}}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"8f15b6dd-f2d2-48e2-a4c9-0230a9ace704","duration":3.7510643005371094,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/671d6e00-6df5-493a-bd9e-8bbf3d77d3ab.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195440Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=96a4f904e8d3ca58d399ab21ade69ec0ea62ff5dfee320c5eeaa6eaa0c4e0561","studio_link":"https://studio.reducto.ai/job/8f15b6dd-f2d2-48e2-a4c9-0230a9ace704","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:40 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:41.233002+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-v3 + retrieval: + chunking: + chunk_mode: page + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/50890748074024915ef138fcb8e0c95f731a0913d223fbe1780bbaaef1ecd835.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/50890748074024915ef138fcb8e0c95f731a0913d223fbe1780bbaaef1ecd835.yaml new file mode 100644 index 00000000000..31f5ed2be1c --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/50890748074024915ef138fcb8e0c95f731a0913d223fbe1780bbaaef1ecd835.yaml @@ -0,0 +1,83 @@ +interactions: +- request: + body: "--43159c32150fa5f506932c4b4a645ef6\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--43159c32150fa5f506932c4b4a645ef6--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=43159c32150fa5f506932c4b4a645ef6 + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://f8bf17e8-ca0b-4add-b484-9982d2e4ac2a.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:50 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://f8bf17e8-ca0b-4add-b484-9982d2e4ac2a.pdf","formatting":{"add_page_markers":true,"table_output_format":"json","merge_tables":true,"include":["change_tracking","highlight","comments"]}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"e3792387-8384-45aa-b02e-a84521db116a","duration":1.0497362613677979,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/f8bf17e8-ca0b-4add-b484-9982d2e4ac2a.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195452Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=7367275a69dfd53f9fcf9f02777f0fe973e7e541fa1871e0347f788b8c5078f4","studio_link":"https://studio.reducto.ai/job/e3792387-8384-45aa-b02e-a84521db116a","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"[[START + OF PAGE 1]]\n\n# Test PDF File\n\n[[END OF PAGE 1]]","embed":"[[START OF PAGE + 1]]\n\n# Test PDF File\n\n[[END OF PAGE 1]]","enriched":null,"enrichment_success":false,"blocks":[{"type":"Page + Number","bbox":{"left":0.0,"top":0.0,"width":0.0,"height":0.0,"page":1,"original_page":1},"content":"[[START + OF PAGE 1]]","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":null},"extra":null},{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null},{"type":"Page + Number","bbox":{"left":0.0,"top":0.0,"width":0.0,"height":0.0,"page":1,"original_page":1},"content":"[[END + OF PAGE 1]]","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":null},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:52 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:53.454592+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + custom_llm_provider: null + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + formatting: + add_page_markers: true + include: + - change_tracking + - highlight + - comments + merge_tables: true + table_output_format: json + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/7cefa9e08179db20d45e255d269e4239015b9b44376ce49c921f8fc057fb9c6a.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/7cefa9e08179db20d45e255d269e4239015b9b44376ce49c921f8fc057fb9c6a.yaml new file mode 100644 index 00000000000..02bd747a4d3 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/7cefa9e08179db20d45e255d269e4239015b9b44376ce49c921f8fc057fb9c6a.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: "--ba392e7bb7694af286b0acfe46365a0f\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--ba392e7bb7694af286b0acfe46365a0f--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=ba392e7bb7694af286b0acfe46365a0f + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://a6d3c3bb-a6f1-4636-8b3a-ee920b29d387.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:46 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://a6d3c3bb-a6f1-4636-8b3a-ee920b29d387.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"4a78c975-1865-46fe-8f89-bea92a49e215","duration":1.1194427013397217,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/6ab9bcf4-1893-4b37-bddb-acda8ce45dfb.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195445Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=1db61d33f602f13e8444e8580b737b31f346e47493cc7eca7c4fcddbde0dfb43","studio_link":"https://studio.reducto.ai/job/d345549f-5d98-4c62-b3e3-24c990144df4","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:47 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:47.967951+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + custom_llm_provider: reducto + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/8270ee13cdfb2ceefd8302a1880653fe30bde512da747d2fd8e78759ccdc9b90.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/8270ee13cdfb2ceefd8302a1880653fe30bde512da747d2fd8e78759ccdc9b90.yaml new file mode 100644 index 00000000000..2bc911985eb --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/8270ee13cdfb2ceefd8302a1880653fe30bde512da747d2fd8e78759ccdc9b90.yaml @@ -0,0 +1,70 @@ +interactions: +- request: + body: "--0f6127f8c781afd158616115a5ebdece\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--0f6127f8c781afd158616115a5ebdece--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=0f6127f8c781afd158616115a5ebdece + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://6ab9bcf4-1893-4b37-bddb-acda8ce45dfb.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:44 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://6ab9bcf4-1893-4b37-bddb-acda8ce45dfb.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"d345549f-5d98-4c62-b3e3-24c990144df4","duration":1.1194427013397217,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/6ab9bcf4-1893-4b37-bddb-acda8ce45dfb.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195445Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=1db61d33f602f13e8444e8580b737b31f346e47493cc7eca7c4fcddbde0dfb43","studio_link":"https://studio.reducto.ai/job/d345549f-5d98-4c62-b3e3-24c990144df4","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:46 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:46.694836+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + custom_llm_provider: null + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/997cd381eb3019b71086e4fdab1b98a3a0d251a27c3bb80178580a711c111e94.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/997cd381eb3019b71086e4fdab1b98a3a0d251a27c3bb80178580a711c111e94.yaml new file mode 100644 index 00000000000..622db8dee4a --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/997cd381eb3019b71086e4fdab1b98a3a0d251a27c3bb80178580a711c111e94.yaml @@ -0,0 +1,100 @@ +interactions: +- request: + body: "--22b7cb49f85a1ae5113a6328b0381ffb\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--22b7cb49f85a1ae5113a6328b0381ffb--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=22b7cb49f85a1ae5113a6328b0381ffb + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://5df745f4-2877-4cf6-9bde-a2f829c93eea.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:48 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://5df745f4-2877-4cf6-9bde-a2f829c93eea.pdf","formatting":{"add_page_markers":false,"table_output_format":"json","merge_tables":true,"include":["signatures","ignore_watermarks"]},"retrieval":{"chunking":{"chunk_mode":"variable","chunk_size":1500,"chunk_overlap":32},"filter_blocks":["Figure","Table","Key + Value"],"embedding_optimized":false},"settings":{"ocr_system":"legacy","extraction_mode":"hybrid","force_url_result":false,"return_ocr_data":false,"return_images":[],"embed_pdf_metadata":false,"embed_pdf_metadata_dpi":100,"persist_results":false,"timeout":900.0,"page_range":[1]}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"7ca67242-1677-484e-b2ef-b256dcdd1ea4","duration":1.3359308242797852,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/5df745f4-2877-4cf6-9bde-a2f829c93eea.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195449Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=077a6551201d2562fd1543fd5c391541cffe606d64307a3f6d103cf18219f80a","studio_link":"https://studio.reducto.ai/job/7ca67242-1677-484e-b2ef-b256dcdd1ea4","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:50 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:50.708637+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + custom_llm_provider: null + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + formatting: + add_page_markers: false + include: + - signatures + - ignore_watermarks + merge_tables: true + table_output_format: json + model: reducto/parse-v3 + retrieval: + chunking: + chunk_mode: variable + chunk_overlap: 32 + chunk_size: 1500 + embedding_optimized: false + filter_blocks: + - Figure + - Table + - Key Value + settings: + embed_pdf_metadata: false + embed_pdf_metadata_dpi: 100 + extraction_mode: hybrid + force_url_result: false + ocr_system: legacy + page_range: + - 1 + persist_results: false + return_images: [] + return_ocr_data: false + timeout: 900.0 + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f2b260eca76347d5982b96c79563999653d141626ca1eadfdb47fc20d645b9d5.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f2b260eca76347d5982b96c79563999653d141626ca1eadfdb47fc20d645b9d5.yaml new file mode 100644 index 00000000000..4515e0a0823 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f2b260eca76347d5982b96c79563999653d141626ca1eadfdb47fc20d645b9d5.yaml @@ -0,0 +1,72 @@ +interactions: +- request: + body: "--7108360c769817ee2464c4729615e289\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--7108360c769817ee2464c4729615e289--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=7108360c769817ee2464c4729615e289 + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://fc6ebcb1-95d4-46ec-90d0-efca33ef197f.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:41 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://fc6ebcb1-95d4-46ec-90d0-efca33ef197f.pdf","settings":{"return_ocr_data":true}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"3f6ef64d-c949-4b8d-9898-afb3b01669e6","duration":1.4480743408203125,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/fc6ebcb1-95d4-46ec-90d0-efca33ef197f.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195443Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=0965ca129cee49a41650df152057a17a4f9e265f92f2261a4f6576587272eb03","studio_link":"https://studio.reducto.ai/job/3f6ef64d-c949-4b8d-9898-afb3b01669e6","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":{"words":[{"text":"Test","bbox":{"left":0.16189475464665032,"top":0.09491921434498797,"width":0.03832089043910207,"height":0.021019531018806225,"page":1,"original_page":1},"confidence":1.0,"chunk_index":null,"rotation":359},{"text":"PDF","bbox":{"left":0.20548195932425706,"top":0.09514990719881924,"width":0.03939929039649714,"height":0.02102524343163076,"page":1,"original_page":1},"confidence":1.0,"chunk_index":null,"rotation":359},{"text":"File","bbox":{"left":0.25014757642558977,"top":0.09538630283240115,"width":0.03177201514150582,"height":0.020984871218902895,"page":1,"original_page":1},"confidence":1.0,"chunk_index":null,"rotation":359}],"lines":[{"text":"Test + PDF File","bbox":{"left":0.16189475464665032,"top":0.09491921434498797,"width":0.12002483692044526,"height":0.021451959706316092,"page":1,"original_page":1},"confidence":1.0,"chunk_index":null,"rotation":359}]},"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:43 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:43.968997+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-v3 + settings: + return_ocr_data: true + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f317c1c7b57a6ac742de1b97f2f84da6aa7892f8cdd4f22158ba4c700dbe8a81.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f317c1c7b57a6ac742de1b97f2f84da6aa7892f8cdd4f22158ba4c700dbe8a81.yaml new file mode 100644 index 00000000000..3349f3f6b2d --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f317c1c7b57a6ac742de1b97f2f84da6aa7892f8cdd4f22158ba4c700dbe8a81.yaml @@ -0,0 +1,39 @@ +interactions: +- request: + body: '{"input":"reducto://invalid-document-for-parity"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"error":{"code":404,"name":"NOT_FOUND","message":"Document ''The file + may have expired or been deleted. Please re-upload and try again.'' not found"},"detail":"Document + ''The file may have expired or been deleted. Please re-upload and try again.'' + not found"}' + headers: + Content-Type: + - application/json + Date: + - Wed, 02 Sep 2026 01:14:02 GMT + status: + code: 404 + message: '' +recorded_at: '2026-09-02T01:14:02.538070+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: reducto://invalid-document-for-parity + type: document_url + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f55e3b684697b9f478393e35202e92b04e51b1ea8b74eaa4a4382a01154b5f5a.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f55e3b684697b9f478393e35202e92b04e51b1ea8b74eaa4a4382a01154b5f5a.yaml new file mode 100644 index 00000000000..5731fa1c48d --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f55e3b684697b9f478393e35202e92b04e51b1ea8b74eaa4a4382a01154b5f5a.yaml @@ -0,0 +1,71 @@ +interactions: +- request: + body: "--04abc28545d4b46a8ee703e56d88cc0c\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--04abc28545d4b46a8ee703e56d88cc0c--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=04abc28545d4b46a8ee703e56d88cc0c + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://f3cc3c72-614a-4104-bb70-a516f37148e0.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:30 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://f3cc3c72-614a-4104-bb70-a516f37148e0.pdf","formatting":{"table_output_format":"md"}}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"aba924dc-7773-4dff-a4a8-3549366ea9fa","duration":4.5764172077178955,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/f3cc3c72-614a-4104-bb70-a516f37148e0.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195435Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=7d4c9464bb3a1a9d5c7a5f957bc10813fb16fcc6de3fea2054a40b469ffa79b3","studio_link":"https://studio.reducto.ai/job/aba924dc-7773-4dff-a4a8-3549366ea9fa","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:35 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:35.993066+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + formatting: + table_output_format: md + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f847dae1a1f6c3777ce3b65421a97f703d67d234593ac188f948142190fdf06d.yaml b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f847dae1a1f6c3777ce3b65421a97f703d67d234593ac188f948142190fdf06d.yaml new file mode 100644 index 00000000000..02298f5a2f0 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/data/reducto-v3/f847dae1a1f6c3777ce3b65421a97f703d67d234593ac188f948142190fdf06d.yaml @@ -0,0 +1,69 @@ +interactions: +- request: + body: "--b7daa0c5be8f80d33a4eb7a54318d93a\r\nContent-Disposition: form-data; name=\"file\"; + filename=\"document\"\r\nContent-Type: application/pdf\r\n\r\n%PDF-1.4\n1 0 + obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n2 0 obj\n<< /Type /Pages /Kids + [3 0 R] /Count 1 >>\nendobj\n3 0 obj\n<< /Type /Page /Parent 2 0 R /Resources + 4 0 R /MediaBox [0 0 612 792] /Contents 5 0 R >>\nendobj\n4 0 obj\n<< /Font + << /F1 << /Type /Font /Subtype /Type1 /BaseFont /Helvetica >> >> >>\nendobj\n5 + 0 obj\n<< /Length 44 >>\nstream\nBT\n/F1 12 Tf\n100 700 Td\n(Test PDF File) + Tj\nET\nendstream\nendobj\nxref\n0 6\n0000000000 65535 f \n0000000009 00000 + n \n0000000058 00000 n \n0000000115 00000 n \n0000000214 00000 n \n0000000293 + 00000 n \ntrailer\n<< /Size 6 /Root 1 0 R >>\nstartxref\n385\n%%EOF\n\r\n--b7daa0c5be8f80d33a4eb7a54318d93a--\r\n" + headers: + Accept: + - '*/*' + Content-Type: + - multipart/form-data; boundary=b7daa0c5be8f80d33a4eb7a54318d93a + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/upload + response: + body: + string: '{"file_id":"reducto://010b01b2-83bd-446d-af11-d120c5ac2c02.pdf","presigned_url":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:27 GMT + status: + code: 200 + message: '' +- request: + body: '{"input":"reducto://010b01b2-83bd-446d-af11-d120c5ac2c02.pdf"}' + headers: + Accept: + - '*/*' + Content-Type: + - application/json + User-Agent: + - litellm/1.101.0 + method: POST + uri: http://parity-provider.invalid/parse + response: + body: + string: '{"response_type":"parse","job_id":"8138a6c6-b726-4300-90b4-e6d5d43f6f70","duration":1.2648272514343262,"pdf_url":"https://prod-storage20241010144745140900000001.s3.amazonaws.com/org/8bd983d1-47dd-440b-93d8-431412d819bf/010b01b2-83bd-446d-af11-d120c5ac2c02.pdf?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIA2UOK6OVBOUYL7WYA%2F20260901%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20260901T195428Z&X-Amz-Expires=43200&X-Amz-SignedHeaders=host&X-Amz-Signature=3cda9020fefff85d9dd13f124908d7b00e55b6e82a72039546d262099cfcf3b4","studio_link":"https://studio.reducto.ai/job/8138a6c6-b726-4300-90b4-e6d5d43f6f70","usage":{"num_pages":1,"credits":1.0,"credit_breakdown":{"page":1.0},"page_billing_breakdown":{"1":["page"]},"non_empty_cell_count":null},"result":{"type":"full","chunks":[{"content":"# + Test PDF File","embed":"# Test PDF File","enriched":null,"enrichment_success":false,"blocks":[{"type":"Title","bbox":{"left":0.16189574527182748,"top":0.09479295553814121,"width":0.11946290650820875,"height":0.02146414301710507,"page":1,"original_page":1},"content":"Test + PDF File","image_url":null,"chart_data":null,"confidence":"high","granular_confidence":{"extract_confidence":null,"parse_confidence":0.7185355693101882},"extra":null}]}],"ocr":null,"custom":null},"parse_mode":null,"document_properties":null}' + headers: + Content-Type: + - application/json + Date: + - Tue, 01 Sep 2026 19:54:29 GMT + status: + code: 200 + message: '' +recorded_at: '2026-09-01T19:54:30.255600+00:00' +ttl_seconds: 0 +version: 1 +x-litellm: + case: + litellm_input: + contract: reducto_v3 + document: + document_url: data:application/pdf;base64,JVBERi0xLjQKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyAvUGFnZXMgMiAwIFIgPj4KZW5kb2JqCjIgMCBvYmoKPDwgL1R5cGUgL1BhZ2VzIC9LaWRzIFszIDAgUl0gL0NvdW50IDEgPj4KZW5kb2JqCjMgMCBvYmoKPDwgL1R5cGUgL1BhZ2UgL1BhcmVudCAyIDAgUiAvUmVzb3VyY2VzIDQgMCBSIC9NZWRpYUJveCBbMCAwIDYxMiA3OTJdIC9Db250ZW50cyA1IDAgUiA+PgplbmRvYmoKNCAwIG9iago8PCAvRm9udCA8PCAvRjEgPDwgL1R5cGUgL0ZvbnQgL1N1YnR5cGUgL1R5cGUxIC9CYXNlRm9udCAvSGVsdmV0aWNhID4+ID4+ID4+CmVuZG9iago1IDAgb2JqCjw8IC9MZW5ndGggNDQgPj4Kc3RyZWFtCkJUCi9GMSAxMiBUZgoxMDAgNzAwIFRkCihUZXN0IFBERiBGaWxlKSBUagpFVAplbmRzdHJlYW0KZW5kb2JqCnhyZWYKMCA2CjAwMDAwMDAwMDAgNjU1MzUgZiAKMDAwMDAwMDAwOSAwMDAwMCBuIAowMDAwMDAwMDU4IDAwMDAwIG4gCjAwMDAwMDAxMTUgMDAwMDAgbiAKMDAwMDAwMDIxNCAwMDAwMCBuIAowMDAwMDAwMjkzIDAwMDAwIG4gCnRyYWlsZXIKPDwgL1NpemUgNiAvUm9vdCAxIDAgUiA+PgpzdGFydHhyZWYKMzg1CiUlRU9GCg== + type: document_url + model: reducto/parse-v3 + request_source: python_replay + schema_version: 1 diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/migrate.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/migrate.py new file mode 100644 index 00000000000..08f3cc66a42 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/migrate.py @@ -0,0 +1,63 @@ +from __future__ import annotations + +import argparse +from pathlib import Path +from typing import Final, cast + +import litellm +from litellm.rust_bridge.ocr import use_litellm_rust +from ......shared.parity.fixtures.recording import ( + RecordedInteraction, + UpstreamEndpoint, + record_upstream_interactions, +) +from ......shared.parity.fixtures.store import FixtureEnvelope, read_fixture, save_fixture +from ......shared.parity.replay import replay_server +from .common import OcrSdkCall +from .config import configured_fixture_directory +from .models import OcrParityCase, OcrSdkInput + + +def _invoke(provider_url: str, case_input: OcrSdkInput) -> object: + sdk_call: Final = cast(OcrSdkCall, litellm.ocr) + return sdk_call(api_base=provider_url, api_key="test-key", **case_input.as_sdk_kwargs()) + + +def migrate_fixture(path: Path) -> Path: + case: Final = read_fixture(path, OcrParityCase) + envelope: Final = FixtureEnvelope.model_validate_json(path.read_text(encoding="utf-8")) + with replay_server() as provider: + for response in case.provider_responses: + provider.enqueue_response(response) + captured: Final = record_upstream_interactions(UpstreamEndpoint(provider.url), case.litellm_input, _invoke) + provider.take_requests(len(case.provider_responses)) + interactions: Final = tuple( + RecordedInteraction(item.request, response) + for item, response in zip(captured, case.provider_responses, strict=True) + ) + destination: Final = save_fixture( + path.parent, + case.litellm_input, + case, + interactions, + recorded_at=envelope.recorded_at, + request_source="python_replay", + ) + read_fixture(destination, OcrParityCase) + path.unlink() + return destination + + +def main() -> None: + parser: Final = argparse.ArgumentParser() + parser.add_argument("--fixture-dir", type=Path, default=configured_fixture_directory()) + args: Final = parser.parse_args() + directory: Final = cast(Path, args.fixture_dir) + use_litellm_rust(False, ocr=None, aocr=None) + paths: Final = tuple(sorted(directory.rglob("*.json"))) + for path in paths: + print(f"Migrated {path.name} to {migrate_fixture(path).name}") + + +if __name__ == "__main__": + main() diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/mistral.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/mistral.py new file mode 100644 index 00000000000..0bdf18b3f76 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/mistral.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final, Literal, cast + +from hypothesis import strategies as st +from hypothesis.strategies import SearchStrategy +from pydantic import model_validator +from typing_extensions import Self + +from ......shared.parity.fixtures.recording import UpstreamEndpoint +from .base import ( + JsonSchemaResponseFormat, + OcrDocument, + OcrSdkInputBase, +) +from .common import ( + OcrFixtureClient, + OcrRecordingTarget, + annotation_format, + document_transport_strategy, + invoke_with_api_key, + pdf_document, +) + +MistralModel = Literal[ + "mistral/mistral-ocr-3", + "mistral/mistral-ocr-3-0", + "mistral/mistral-ocr-2512", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-latest", + "mistral-ocr-3", + "mistral-ocr-3-0", + "mistral-ocr-2512", + "mistral-ocr-4-0", + "mistral-ocr-4-1", + "mistral-ocr-4", + "mistral-ocr-latest", +] +MistralFixtureModel = MistralModel | Literal["mistral/invalid-ocr-model-for-parity"] + +MISTRAL_MODELS: Final[tuple[MistralModel, ...]] = ( + "mistral/mistral-ocr-3", + "mistral/mistral-ocr-3-0", + "mistral/mistral-ocr-2512", + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-latest", +) + + +class MistralCompatibleOcrSdkInput(OcrSdkInputBase): + document: OcrDocument + pages: str | list[int] | None = None + include_image_base64: bool | None = None + image_limit: int | None = None + image_min_size: int | None = None + bbox_annotation_format: JsonSchemaResponseFormat | None = None + document_annotation_format: JsonSchemaResponseFormat | None = None + document_annotation_prompt: str | None = None + extract_header: bool = False + extract_footer: bool = False + table_format: Literal["markdown", "html"] | None = None + confidence_scores_granularity: Literal["page", "word", "block"] | None = None + include_blocks: bool = True + id: str | None = None + + @model_validator(mode="after") + def validate_annotation_prompt(self) -> Self: + if self.document_annotation_prompt is not None and self.document_annotation_format is None: + raise ValueError("document_annotation_prompt requires document_annotation_format") + return self + + +class MistralOcrSdkInput(MistralCompatibleOcrSdkInput): + contract: Literal["mistral"] = "mistral" + model: MistralFixtureModel + custom_llm_provider: Literal["mistral"] | None = None + + @model_validator(mode="after") + def validate_provider_routing(self) -> Self: + if not self.model.startswith("mistral/") and self.custom_llm_provider != "mistral": + raise ValueError("unqualified Mistral models require custom_llm_provider='mistral'") + return self + + +MISTRAL_MODEL: Final[MistralModel] = "mistral/mistral-ocr-latest" +MISTRAL_PROVIDER_REJECTED_INPUTS: Final[tuple[MistralOcrSdkInput, ...]] = ( + MistralOcrSdkInput( + model="mistral/invalid-ocr-model-for-parity", + document=pdf_document(), + ), +) +MistralFeatureLevel = Literal["2505", "2512", "4"] +_MISTRAL_4_MODELS: Final = frozenset( + { + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-latest", + } +) +_MISTRAL_2512_MODELS: Final = frozenset( + {*_MISTRAL_4_MODELS, "mistral/mistral-ocr-2512", "mistral/mistral-ocr-3", "mistral/mistral-ocr-3-0"} +) + + +def _feature_level(model: str) -> MistralFeatureLevel: + if model in _MISTRAL_4_MODELS: + return "4" + if model in _MISTRAL_2512_MODELS: + return "2512" + return "2505" + + +def _optional_param_strategies( + *, + include_document_annotation_prompt: bool = True, +) -> tuple[ + tuple[SearchStrategy[dict[str, object]], ...], + tuple[SearchStrategy[dict[str, object]], ...], + tuple[SearchStrategy[dict[str, object]], ...], +]: + annotation: Final = annotation_format("document_title") + common: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( + st.sampled_from(((0,), (0, 1))).map(list).map(lambda value: {"pages": value}), + st.sampled_from((False, True)).map(lambda value: {"include_image_base64": value}), + st.just({"image_limit": 1}), + st.just({"image_min_size": 300}), + st.just({"bbox_annotation_format": annotation_format("bounding_boxes")}), + st.just({"document_annotation_format": annotation}), + *( + ( + st.just( + { + "document_annotation_format": annotation, + "document_annotation_prompt": "Extract the visible title", + } + ), + ) + if include_document_annotation_prompt + else () + ), + st.sampled_from(("page", "word")).map(lambda value: {"confidence_scores_granularity": value}), + ) + feature_2512: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( + st.sampled_from((False, True)).map(lambda value: {"extract_header": value}), + st.sampled_from((False, True)).map(lambda value: {"extract_footer": value}), + st.sampled_from(("markdown", "html")).map(lambda value: {"table_format": value}), + ) + feature_4: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( + st.just({"pages": "0-2"}), + st.sampled_from((False, True)).map(lambda value: {"include_blocks": value}), + st.just({"include_blocks": True, "confidence_scores_granularity": "block"}), + ) + return common, feature_2512, feature_4 + + +def mistral_optional_params_strategy( + feature_level: MistralFeatureLevel, + *, + include_document_annotation_prompt: bool = True, +) -> SearchStrategy[dict[str, object]]: + common, feature_2512, feature_4 = _optional_param_strategies( + include_document_annotation_prompt=include_document_annotation_prompt + ) + return st.one_of( + *common, + *(feature_2512 if feature_level in {"2512", "4"} else ()), + *(feature_4 if feature_level == "4" else ()), + ) + + +def _mistral_input_values( + document: OcrDocument, + optional_params: dict[str, object] | None = None, +) -> dict[str, object]: + return {"document": document, **(optional_params or {})} + + +def _mistral_input( + model: str, + document: OcrDocument, + optional_params: dict[str, object] | None = None, +) -> MistralOcrSdkInput: + return MistralOcrSdkInput.model_validate({"model": model, **_mistral_input_values(document, optional_params)}) + + +def mistral_input_values_strategy( + feature_level: MistralFeatureLevel, + inline_image_data_uri: str, + *, + include_document_annotation_prompt: bool = True, +) -> SearchStrategy[dict[str, object]]: + option_document: Final = pdf_document() + return st.one_of( + document_transport_strategy(inline_image_data_uri).map(_mistral_input_values), + mistral_optional_params_strategy( + feature_level, + include_document_annotation_prompt=include_document_annotation_prompt, + ).map(lambda optional_params: _mistral_input_values(option_document, optional_params)), + ) + + +def mistral_input_strategy( + model: str, + inline_image_data_uri: str, + feature_level: MistralFeatureLevel | None = None, +) -> SearchStrategy[MistralOcrSdkInput]: + return mistral_input_values_strategy(feature_level or _feature_level(model), inline_image_data_uri).map( + lambda values: MistralOcrSdkInput.model_validate({"model": model, **values}) + ) + + +def _mistral_recording_strategy(inline_image_data_uri: str) -> SearchStrategy[MistralOcrSdkInput]: + document: Final = pdf_document() + baseline_models: Final = tuple(model for model in MISTRAL_MODELS if model != MISTRAL_MODEL) + common, feature_2512, feature_4 = _optional_param_strategies() + common_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*common) + feature_2512_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_2512) + feature_4_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_4) + return st.one_of( + st.sampled_from(baseline_models).map(lambda model: _mistral_input(model, document)), + document_transport_strategy(inline_image_data_uri).map( + lambda selected_document: _mistral_input(MISTRAL_MODEL, selected_document) + ), + common_options.map(lambda optional_params: _mistral_input(MISTRAL_MODEL, document, optional_params)), + feature_2512_options.map( + lambda optional_params: _mistral_input("mistral/mistral-ocr-2512", document, optional_params) + ), + feature_4_options.map( + lambda optional_params: _mistral_input("mistral/mistral-ocr-4-1", document, optional_params) + ), + ) + + +def mistral_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("MISTRAL_API_KEY") + if not api_key: + return () + configured: Final = environ.get("MISTRAL_API_BASE", "https://api.mistral.ai").rstrip("/") + base_url: Final = configured.removesuffix("/v1") + return ( + OcrRecordingTarget( + name="mistral-ocr", + upstream=UpstreamEndpoint(base_url=base_url), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + _mistral_recording_strategy(inline_image_data_uri), + ), + invocation=invoke_with_api_key(client, api_key), + required_inputs=MISTRAL_PROVIDER_REJECTED_INPUTS, + ), + ) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/models.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/models.py new file mode 100644 index 00000000000..bd0e9736245 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/models.py @@ -0,0 +1,71 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Annotated, Final, cast + +from pydantic import Field, model_validator + +from ......shared.parity.fixture_models import ParityCase +from .azure import ( + AzureDocumentIntelligenceOcrSdkInput, + AzureMistralOcrSdkInput, +) +from .mistral import MistralOcrSdkInput +from .reducto import ReductoParseLegacySdkInput, ReductoParseV3SdkInput +from .vertex import VertexDeepSeekOcrSdkInput, VertexMistralOcrSdkInput + +__all__ = ("OcrParityCase", "OcrSdkInput") + + +OcrSdkInput = Annotated[ + MistralOcrSdkInput + | AzureMistralOcrSdkInput + | VertexMistralOcrSdkInput + | AzureDocumentIntelligenceOcrSdkInput + | VertexDeepSeekOcrSdkInput + | ReductoParseV3SdkInput + | ReductoParseLegacySdkInput, + Field(discriminator="contract"), +] + + +class OcrParityCase(ParityCase[OcrSdkInput]): + @model_validator(mode="before") + @classmethod + def load_legacy_contract(cls, value: object) -> object: + if not isinstance(value, Mapping): + return value + fixture: Final = cast(Mapping[str, object], value) + litellm_input: Final = fixture.get("litellm_input") + if not isinstance(litellm_input, Mapping) or "contract" in litellm_input: + return fixture + legacy_input: Final = cast(Mapping[str, object], litellm_input) + legacy_contract: Final = legacy_input.get("boundary") + if isinstance(legacy_contract, str): + return { + **fixture, + "litellm_input": { + "contract": legacy_contract, + **{key: item for key, item in legacy_input.items() if key != "boundary"}, + }, + } + model: Final = legacy_input.get("model") + if not isinstance(model, str): + return fixture + return {**fixture, "litellm_input": {"contract": _legacy_contract(model), **legacy_input}} + + +def _legacy_contract(model: str) -> str: + if model.startswith("azure_ai/doc-intelligence/"): + return "azure_document_intelligence" + if model.startswith("azure_ai/"): + return "azure_mistral" + if model.startswith("vertex_ai/deepseek"): + return "vertex_deepseek" + if model.startswith("vertex_ai/"): + return "vertex_mistral" + if model.endswith("parse-v3"): + return "reducto_v3" + if model.endswith("parse-legacy"): + return "reducto_legacy" + return "mistral" diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/record.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/record.py new file mode 100644 index 00000000000..4334cbb8a39 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/record.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import logging +import os +from collections.abc import Mapping +from typing import Final, cast + +from dotenv import load_dotenv + +import litellm +from litellm.rust_bridge.ocr import use_litellm_rust +from ......shared.parity.fixtures.cli import parse_recording_args +from ......shared.parity.fixtures.media import structured_image_data_uri +from ......shared.parity.fixtures.pipeline import record_fixtures +from ......shared.parity.fixtures.store import fixture_directory +from .azure import ( + azure_document_intelligence_recording_targets, + azure_mistral_recording_targets, +) +from .base import OcrSdkInputBase +from .common import OcrFixtureClient, OcrRecordingTarget, OcrSdkCall +from .config import DEFAULT_FIXTURE_DIRECTORY, FIXTURE_DIR_ENV +from .mistral import mistral_recording_targets +from .models import OcrParityCase +from .reducto import reducto_recording_targets +from .vertex import vertex_recording_targets + + +class LiteLLMOcrFixtureClient: + def __init__(self, sdk_call: OcrSdkCall) -> None: + self.sdk_call: Final = sdk_call + + def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: + self.sdk_call(api_base=api_base, api_key=api_key, **case_input.as_sdk_kwargs()) + + +def discover_targets( + environ: Mapping[str, str], + client: OcrFixtureClient, + inline_image_data_uri: str, +) -> tuple[OcrRecordingTarget, ...]: + return ( + *mistral_recording_targets(environ, client, inline_image_data_uri), + *azure_mistral_recording_targets(environ, client, inline_image_data_uri), + *azure_document_intelligence_recording_targets(environ, client), + *vertex_recording_targets(environ, client, inline_image_data_uri), + *reducto_recording_targets(environ, client, inline_image_data_uri), + ) + + +def require_targets(targets: tuple[OcrRecordingTarget, ...]) -> tuple[OcrRecordingTarget, ...]: + if targets: + return targets + raise SystemExit("No OCR fixture providers are configured. Set a supported provider API key and endpoint") + + +def main() -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + load_dotenv() + args: Final = parse_recording_args() + client: Final = LiteLLMOcrFixtureClient(cast(OcrSdkCall, litellm.ocr)) + inline_image_data_uri: Final = structured_image_data_uri() + targets: Final = require_targets(discover_targets(os.environ, client, inline_image_data_uri)) + root: Final = fixture_directory( + args.fixture_dir, + os.environ.get(FIXTURE_DIR_ENV), + DEFAULT_FIXTURE_DIRECTORY, + ) + use_litellm_rust(False, ocr=None, aocr=None) + summary: Final = record_fixtures(targets, root, args.examples, args.concurrency, OcrParityCase) + return summary.exit_code + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/reducto.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/reducto.py new file mode 100644 index 00000000000..fe8fab50518 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/reducto.py @@ -0,0 +1,440 @@ +from __future__ import annotations + +import base64 +import binascii +from collections.abc import Mapping +from typing import Annotated, Final, Literal, cast + +from hypothesis import strategies as st +from hypothesis.strategies import SearchStrategy +from pydantic import Field, field_validator, model_validator +from typing_extensions import Self + +from ......shared.parity.fixture_models import FixtureModel, JsonObject +from ......shared.parity.fixtures.media import structured_pdf_data_uri +from ......shared.parity.fixtures.recording import UpstreamEndpoint +from .base import OcrSdkInputBase +from .common import ( + OcrFixtureClient, + OcrRecordingTarget, + image_data_document, + invoke_with_api_key, +) + + +def _validate_reducto_source(source: str) -> str: + if source.startswith("reducto://"): + return source + if not source.startswith("data:"): + raise ValueError("Reducto documents require a reducto:// id or base64 data URI") + try: + header, encoded = source.split(",", 1) + except ValueError as error: + raise ValueError("invalid Reducto data URI") from error + if ";base64" not in header: + raise ValueError("Reducto data URIs must be base64 encoded") + try: + base64.b64decode(encoded, validate=True) + except (binascii.Error, ValueError) as error: + raise ValueError("invalid Reducto base64 payload") from error + return source + + +class ReductoImageUrlDocument(FixtureModel): + type: Literal["image_url"] + image_url: str + + @field_validator("image_url") + @classmethod + def validate_image_url(cls, value: str) -> str: + return _validate_reducto_source(value) + + +class ReductoDocumentUrlDocument(FixtureModel): + type: Literal["document_url"] + document_url: str + + @field_validator("document_url") + @classmethod + def validate_document_url(cls, value: str) -> str: + return _validate_reducto_source(value) + + +ReductoDocument = Annotated[ + ReductoImageUrlDocument | ReductoDocumentUrlDocument, + Field(discriminator="type"), +] + +ReductoTableOutputFormat = Literal["html", "json", "md", "jsonbbox", "dynamic", "csv"] +ReductoReturnImage = Literal["figure", "table", "page"] +ReductoFormattingInclude = Literal[ + "change_tracking", + "highlight", + "comments", + "hyperlinks", + "signatures", + "ignore_watermarks", +] +ReductoBlockType = Literal[ + "Header", + "Footer", + "Title", + "Section Header", + "Page Number", + "List Item", + "Figure", + "Table", + "Key Value", + "Text", + "Comment", + "Signature", +] +_REDUCTO_FILTER_BLOCK_GROUPS: Final[tuple[tuple[ReductoBlockType, ...], ...]] = ( + (), + ("Header",), + ("Header", "Footer", "Page Number"), + ("Figure", "Table", "Key Value"), +) +_REDUCTO_RETURN_IMAGE_GROUPS: Final[tuple[tuple[ReductoReturnImage, ...], ...]] = ( + (), + ("figure",), + ("table",), + ("page",), + ("figure", "table"), +) + + +class ReductoFormatting(FixtureModel): + add_page_markers: bool = False + table_output_format: ReductoTableOutputFormat = "dynamic" + merge_tables: bool = False + include: list[ReductoFormattingInclude] = Field(default_factory=list) + + @field_validator("include") + @classmethod + def validate_unique_include(cls, value: list[ReductoFormattingInclude]) -> list[ReductoFormattingInclude]: + if len(value) != len(set(value)): + raise ValueError("formatting.include entries must be unique") + return value + + +class ReductoChunking(FixtureModel): + chunk_mode: Literal["variable", "section", "page", "disabled", "block", "page_sections"] = "disabled" + chunk_size: int | None = None + chunk_overlap: int = Field(default=0, ge=0) + + @model_validator(mode="after") + def validate_chunking(self) -> Self: + if self.chunk_size is not None and self.chunk_size <= 0: + raise ValueError("chunk_size must be positive") + if self.chunk_size is not None and self.chunk_overlap >= self.chunk_size: + raise ValueError("chunk_overlap must be less than chunk_size") + return self + + +class ReductoRetrieval(FixtureModel): + chunking: ReductoChunking = Field(default_factory=ReductoChunking) + filter_blocks: list[ReductoBlockType] = Field(default_factory=list) + embedding_optimized: bool = False + + @field_validator("filter_blocks") + @classmethod + def validate_unique_blocks(cls, value: list[ReductoBlockType]) -> list[ReductoBlockType]: + if len(value) != len(set(value)): + raise ValueError("retrieval.filter_blocks entries must be unique") + return value + + +class ReductoPageRange(FixtureModel): + start: int | None = Field(default=None, ge=1) + end: int | None = Field(default=None, ge=1) + + @model_validator(mode="after") + def validate_range(self) -> Self: + if self.start is not None and self.end is not None and self.end < self.start: + raise ValueError("page range end must be greater than or equal to start") + return self + + +class ReductoTenantThrottling(FixtureModel): + tenant_id: str = Field(min_length=1, max_length=256) + max_share: float = Field(default=0.5, gt=0, le=1) + + +class ReductoHybridVpcSettings(FixtureModel): + environment: str | None = None + + +ReductoPageSelection = ReductoPageRange | list[ReductoPageRange] | list[int] | list[str] +ReductoV3Model = Literal["reducto/parse-v3", "parse-v3"] +ReductoLegacyModel = Literal["reducto/parse-legacy", "parse-legacy"] +_ReductoV3Route = Literal["qualified", "image", "unqualified"] +_ReductoLegacyRoute = Literal["qualified", "unqualified"] + +REDUCTO_V3_MODELS: Final[tuple[Literal["reducto/parse-v3"], ...]] = ("reducto/parse-v3",) +REDUCTO_LEGACY_MODELS: Final[tuple[Literal["reducto/parse-legacy"], ...]] = ("reducto/parse-legacy",) + + +class ReductoSettings(FixtureModel): + model: Literal["r-1"] | None = None + ocr_system: Literal["standard", "legacy"] = "standard" + extraction_mode: Literal["ocr", "hybrid", "metadata"] = "hybrid" + force_url_result: bool = False + force_file_extension: str | None = None + return_ocr_data: bool = False + return_images: list[ReductoReturnImage] = Field(default_factory=list) + embed_pdf_metadata: bool = False + embed_pdf_metadata_dpi: int = Field(default=100, ge=50, le=250) + persist_results: bool = False + tenant_throttling: ReductoTenantThrottling | None = None + timeout: float | None = Field(default=None, gt=0) + page_range: ReductoPageSelection | None = None + document_password: str | None = None + hybrid_vpc: ReductoHybridVpcSettings = Field(default_factory=ReductoHybridVpcSettings) + + @field_validator("return_images") + @classmethod + def validate_unique_images(cls, value: list[ReductoReturnImage]) -> list[ReductoReturnImage]: + if len(value) != len(set(value)): + raise ValueError("settings.return_images entries must be unique") + return value + + +class ReductoParseV3SdkInput(OcrSdkInputBase): + contract: Literal["reducto_v3"] = "reducto_v3" + model: ReductoV3Model + document: ReductoDocument + custom_llm_provider: Literal["reducto"] | None = None + formatting: ReductoFormatting = Field(default_factory=ReductoFormatting) + retrieval: ReductoRetrieval = Field(default_factory=ReductoRetrieval) + settings: ReductoSettings = Field(default_factory=ReductoSettings) + + @model_validator(mode="after") + def validate_provider_routing(self) -> Self: + if self.model == "parse-v3" and self.custom_llm_provider != "reducto": + raise ValueError("unqualified Reducto models require custom_llm_provider='reducto'") + return self + + +class ReductoParseLegacySdkInput(OcrSdkInputBase): + contract: Literal["reducto_legacy"] = "reducto_legacy" + model: ReductoLegacyModel + document: ReductoDocument + custom_llm_provider: Literal["reducto"] | None = None + enhance: JsonObject | None = None + + @model_validator(mode="after") + def validate_provider_routing(self) -> Self: + if self.model == "parse-legacy" and self.custom_llm_provider != "reducto": + raise ValueError("unqualified Reducto models require custom_llm_provider='reducto'") + return self + + +_REDUCTO_PROVIDER_REJECTED_DOCUMENT: Final = ReductoDocumentUrlDocument( + type="document_url", + document_url="reducto://invalid-document-for-parity", +) +REDUCTO_V3_PROVIDER_REJECTED_INPUTS: Final[tuple[ReductoParseV3SdkInput, ...]] = ( + ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=_REDUCTO_PROVIDER_REJECTED_DOCUMENT, + ), +) +REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS: Final[tuple[ReductoParseLegacySdkInput, ...]] = ( + ReductoParseLegacySdkInput( + model="reducto/parse-legacy", + document=_REDUCTO_PROVIDER_REJECTED_DOCUMENT, + ), +) + + +_REDUCTO_API_BASE: Final = "https://platform.reducto.ai" + + +def _formatting_strategy() -> SearchStrategy[ReductoFormatting]: + values: Final = st.one_of( + st.sampled_from(("dynamic", "html", "md", "json", "csv", "jsonbbox")).map( + lambda value: {"table_output_format": value} + ), + st.sampled_from((False, True)).map(lambda value: {"add_page_markers": value}), + st.sampled_from((False, True)).map(lambda value: {"merge_tables": value}), + st.sampled_from( + ( + (), + ("hyperlinks",), + ("change_tracking", "highlight", "comments"), + ("signatures", "ignore_watermarks"), + ) + ) + .map(list) + .map(lambda value: {"include": value}), + ) + return values.map(ReductoFormatting.model_validate) + + +def _chunking_strategy() -> SearchStrategy[ReductoChunking]: + return st.one_of( + st.sampled_from(("disabled", "section", "page", "block", "page_sections")).map( + lambda mode: ReductoChunking(chunk_mode=mode) + ), + st.just(ReductoChunking(chunk_mode="variable")), + st.sampled_from((250, 1000, 1500)).map(lambda size: ReductoChunking(chunk_mode="variable", chunk_size=size)), + st.sampled_from((32, 128)).map( + lambda overlap: ReductoChunking(chunk_mode="variable", chunk_size=1000, chunk_overlap=overlap) + ), + ) + + +def _retrieval_strategy() -> SearchStrategy[ReductoRetrieval]: + filter_blocks: Final = cast( + SearchStrategy[list[ReductoBlockType]], + st.sampled_from(_REDUCTO_FILTER_BLOCK_GROUPS).map(list), + ) + return st.one_of( + _chunking_strategy().map(lambda chunking: ReductoRetrieval(chunking=chunking)), + filter_blocks.map(lambda selected_blocks: ReductoRetrieval(filter_blocks=selected_blocks)), + st.sampled_from((False, True)).map( + lambda optimized: ReductoRetrieval( + chunking=ReductoChunking(chunk_mode="variable"), + embedding_optimized=optimized, + ) + ), + ) + + +def _settings_strategy() -> SearchStrategy[ReductoSettings]: + # force_url_result stays model-compatible but is not recorded until the + # response transform follows and downloads result.url. + return_images: Final[SearchStrategy[list[ReductoReturnImage]]] = st.sampled_from(_REDUCTO_RETURN_IMAGE_GROUPS).map( + list + ) + page_ranges: Final = st.one_of( + st.just(ReductoPageRange(start=1, end=1)), + st.just(ReductoPageRange(start=1, end=3)), + st.sampled_from( + ( + ( + ReductoPageRange(start=1, end=2), + ReductoPageRange(start=4, end=5), + ), + ) + ).map(list), + ) + return st.one_of( + st.just(ReductoSettings(model="r-1")), + st.sampled_from(("standard", "legacy")).map(lambda value: ReductoSettings(ocr_system=value)), + st.sampled_from(("hybrid", "ocr", "metadata")).map(lambda value: ReductoSettings(extraction_mode=value)), + st.just(ReductoSettings(return_ocr_data=True)), + return_images.map(lambda selected_images: ReductoSettings(return_images=selected_images)), + st.just(ReductoSettings(embed_pdf_metadata=True)), + st.sampled_from((50, 100, 250)).map( + lambda dpi: ReductoSettings(embed_pdf_metadata=True, embed_pdf_metadata_dpi=dpi) + ), + st.just(ReductoSettings(timeout=300.0)), + page_ranges.map(lambda page_range: ReductoSettings(page_range=page_range)), + ) + + +def _reducto_v3_baseline( + route: _ReductoV3Route, + document: ReductoDocument, + inline_image_data_uri: str, +) -> ReductoParseV3SdkInput: + if route == "image": + inline_image: Final = ReductoImageUrlDocument.model_validate( + image_data_document(inline_image_data_uri).model_dump(mode="json") + ) + return ReductoParseV3SdkInput(model="reducto/parse-v3", document=inline_image) + if route == "unqualified": + return ReductoParseV3SdkInput( + model="parse-v3", + custom_llm_provider="reducto", + document=document, + ) + return ReductoParseV3SdkInput(model="reducto/parse-v3", document=document) + + +def reducto_v3_input_strategy( + inline_image_data_uri: str, + document: ReductoDocument | None = None, +) -> SearchStrategy[ReductoParseV3SdkInput]: + selected_document: Final = document or ReductoDocumentUrlDocument( + type="document_url", document_url="reducto://fixture-document.pdf" + ) + baseline_routes: Final[tuple[_ReductoV3Route, ...]] = ("qualified", "image", "unqualified") + return st.one_of( + st.sampled_from(baseline_routes).map( + lambda route: _reducto_v3_baseline(route, selected_document, inline_image_data_uri) + ), + _formatting_strategy().map( + lambda formatting: ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=selected_document, + formatting=formatting, + ) + ), + _retrieval_strategy().map( + lambda retrieval: ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=selected_document, + retrieval=retrieval, + ) + ), + _settings_strategy().map( + lambda settings: ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=selected_document, + settings=settings, + ) + ), + ) + + +def _reducto_legacy_input( + route: _ReductoLegacyRoute, + document: ReductoDocument, +) -> ReductoParseLegacySdkInput: + if route == "unqualified": + return ReductoParseLegacySdkInput( + model="parse-legacy", + custom_llm_provider="reducto", + document=document, + ) + return ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=document) + + +def reducto_legacy_input_strategy( + document: ReductoDocument | None = None, +) -> SearchStrategy[ReductoParseLegacySdkInput]: + selected_document: Final = document or ReductoDocumentUrlDocument( + type="document_url", document_url="reducto://fixture-document.pdf" + ) + routes: Final[tuple[_ReductoLegacyRoute, ...]] = ("qualified", "unqualified") + return st.sampled_from(routes).map(lambda route: _reducto_legacy_input(route, selected_document)) + + +def reducto_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("REDUCTO_API_KEY") + if not api_key: + return () + base_url: Final = environ.get("REDUCTO_API_BASE", _REDUCTO_API_BASE).rstrip("/") + document: Final = ReductoDocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri()) + invocation: Final = invoke_with_api_key(client, api_key) + return ( + OcrRecordingTarget( + name="reducto-v3", + upstream=UpstreamEndpoint(base_url=base_url), + strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_v3_input_strategy(inline_image_data_uri, document)), + invocation=invocation, + required_inputs=REDUCTO_V3_PROVIDER_REJECTED_INPUTS, + ), + OcrRecordingTarget( + name="reducto-legacy", + upstream=UpstreamEndpoint(base_url=base_url), + strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_legacy_input_strategy(document)), + invocation=invocation, + required_inputs=REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, + ), + ) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/vertex.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/vertex.py new file mode 100644 index 00000000000..ced0b0f2481 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/fixtures/vertex.py @@ -0,0 +1,148 @@ +from __future__ import annotations + +from collections.abc import Mapping +from typing import Final, Literal, cast + +from hypothesis import strategies as st +from hypothesis.strategies import DrawFn, SearchStrategy + +from ......shared.parity.fixtures.recording import UpstreamEndpoint +from .base import OcrDocument, OcrSdkInputBase +from .common import ( + OcrFixtureClient, + OcrRecordingTarget, + image_data_document, + invoke_with_api_key, +) +from .mistral import ( + MistralCompatibleOcrSdkInput, + mistral_input_values_strategy, +) + +VertexMistralModel = Literal["vertex_ai/mistral-ocr-2505"] +VertexDeepSeekModel = Literal["vertex_ai/deepseek-ai/deepseek-ocr-maas"] +VertexMistralFixtureModel = VertexMistralModel | Literal["vertex_ai/invalid-ocr-model-for-parity"] +VertexDeepSeekFixtureModel = VertexDeepSeekModel | Literal["vertex_ai/deepseek-ai/invalid-ocr-model-for-parity"] + +VERTEX_MISTRAL_MODELS: Final[tuple[VertexMistralModel, ...]] = ("vertex_ai/mistral-ocr-2505",) +VERTEX_DEEPSEEK_MODELS: Final[tuple[VertexDeepSeekModel, ...]] = ("vertex_ai/deepseek-ai/deepseek-ocr-maas",) + + +class VertexMistralOcrSdkInput(MistralCompatibleOcrSdkInput): + contract: Literal["vertex_mistral"] = "vertex_mistral" + model: VertexMistralFixtureModel = "vertex_ai/mistral-ocr-2505" + custom_llm_provider: Literal["vertex_ai"] | None = None + vertex_project: str + vertex_location: str = "us-central1" + + +class VertexDeepSeekOcrSdkInput(OcrSdkInputBase): + contract: Literal["vertex_deepseek"] = "vertex_deepseek" + model: VertexDeepSeekFixtureModel = "vertex_ai/deepseek-ai/deepseek-ocr-maas" + document: OcrDocument + custom_llm_provider: Literal["vertex_ai"] | None = None + vertex_project: str + vertex_location: str = "us-central1" + + +def vertex_mistral_provider_rejected_inputs( + project: str, + location: str, + inline_image_data_uri: str, +) -> tuple[VertexMistralOcrSdkInput, ...]: + return ( + VertexMistralOcrSdkInput( + model="vertex_ai/invalid-ocr-model-for-parity", + document=image_data_document(inline_image_data_uri), + vertex_project=project, + vertex_location=location, + ), + ) + + +def vertex_deepseek_provider_rejected_inputs( + project: str, + location: str, + inline_image_data_uri: str, +) -> tuple[VertexDeepSeekOcrSdkInput, ...]: + return ( + VertexDeepSeekOcrSdkInput( + model="vertex_ai/deepseek-ai/invalid-ocr-model-for-parity", + document=image_data_document(inline_image_data_uri), + vertex_project=project, + vertex_location=location, + ), + ) + + +def _as_vertex_mistral( + values: dict[str, object], + project: str, + location: str, + model: VertexMistralModel, +) -> VertexMistralOcrSdkInput: + return VertexMistralOcrSdkInput.model_validate( + {**values, "model": model, "vertex_project": project, "vertex_location": location} + ) + + +def vertex_mistral_input_strategy( + project: str, + location: str, + inline_image_data_uri: str, +) -> SearchStrategy[VertexMistralOcrSdkInput]: + return st.builds( + _as_vertex_mistral, + project=st.just(project), + location=st.just(location), + model=st.sampled_from(VERTEX_MISTRAL_MODELS), + values=mistral_input_values_strategy("2505", inline_image_data_uri), + ) + + +@st.composite +def vertex_deepseek_input_strategy( + draw: DrawFn, project: str, location: str, inline_image_data_uri: str +) -> VertexDeepSeekOcrSdkInput: + return VertexDeepSeekOcrSdkInput.model_validate( + { + "model": draw(st.sampled_from(VERTEX_DEEPSEEK_MODELS)), + "document": image_data_document(inline_image_data_uri), + "vertex_project": project, + "vertex_location": location, + } + ) + + +def vertex_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: + api_key: Final = environ.get("VERTEX_AI_API_KEY") + project: Final = environ.get("VERTEXAI_PROJECT") or environ.get("VERTEX_PROJECT") + location: Final = environ.get("VERTEXAI_LOCATION") or environ.get("VERTEX_LOCATION") or "us-central1" + if not api_key or not project: + return () + base_url: Final = environ.get("VERTEX_AI_API_BASE") or f"https://{location}-aiplatform.googleapis.com" + invocation: Final = invoke_with_api_key(client, api_key) + return ( + OcrRecordingTarget( + name="vertex-mistral", + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + vertex_mistral_input_strategy(project, location, inline_image_data_uri), + ), + invocation=invocation, + required_inputs=vertex_mistral_provider_rejected_inputs(project, location, inline_image_data_uri), + ), + OcrRecordingTarget( + name="vertex-deepseek", + upstream=UpstreamEndpoint(base_url=base_url.rstrip("/")), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + vertex_deepseek_input_strategy(project, location, inline_image_data_uri), + ), + invocation=invocation, + required_inputs=vertex_deepseek_provider_rejected_inputs(project, location, inline_image_data_uri), + ), + ) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_models.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_models.py new file mode 100644 index 00000000000..42a24ad410d --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_models.py @@ -0,0 +1,1388 @@ +from __future__ import annotations + +import base64 +from collections.abc import Callable +from datetime import date +from pathlib import Path +from typing import Final, TypeVar, cast +from unittest.mock import patch +from urllib.parse import parse_qs, urlparse + +import httpx +import pytest +import respx +from hypothesis import find, given, settings +from hypothesis import strategies as st +from hypothesis.strategies import DataObject, SearchStrategy +from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationError + +from litellm.llms.azure_ai.ocr.document_intelligence.transformation import AzureDocumentIntelligenceOCRConfig +from litellm.llms.azure_ai.ocr.transformation import AzureAIOCRConfig +from litellm.llms.base_llm.ocr.transformation import ( + BaseOCRConfig, + DocumentType, + OCRRequestData, +) +from litellm.llms.mistral.ocr.transformation import MistralOCRConfig +from litellm.llms.reducto.ocr.transformation import ReductoParseLegacyConfig, ReductoParseV3Config +from litellm.llms.vertex_ai.ocr.deepseek_transformation import VertexAIDeepSeekOCRConfig +from litellm.llms.vertex_ai.ocr.transformation import VertexAIOCRConfig +from .....shared.parity.fixtures.media import structured_pdf_data_uri +from .conftest import ocr_fixture_marks +from .fixtures.azure import ( + AZURE_DOCUMENT_INTELLIGENCE_MODELS, + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, + AZURE_MISTRAL_MODELS, + AzureDocumentIntelligenceOcrSdkInput, + AzureMistralOcrSdkInput, + azure_document_intelligence_input_strategy, + azure_mistral_input_strategy, +) +from .fixtures.base import ( + DocumentUrlDocument, + ImageUrlDocument, + ImageUrlValue, + JsonSchemaDefinition, + JsonSchemaResponseFormat, + OcrSdkInputBase, +) +from .fixtures.mistral import MISTRAL_MODELS, MistralOcrSdkInput, mistral_input_strategy +from .fixtures.models import OcrParityCase, OcrSdkInput +from .fixtures.reducto import ( + REDUCTO_LEGACY_MODELS, + REDUCTO_V3_MODELS, + ReductoChunking, + ReductoDocumentUrlDocument, + ReductoFormatting, + ReductoImageUrlDocument, + ReductoPageRange, + ReductoParseLegacySdkInput, + ReductoParseV3SdkInput, + ReductoRetrieval, + ReductoSettings, + reducto_legacy_input_strategy, + reducto_v3_input_strategy, +) +from .fixtures.vertex import ( + VERTEX_DEEPSEEK_MODELS, + VERTEX_MISTRAL_MODELS, + VertexDeepSeekOcrSdkInput, + VertexMistralOcrSdkInput, + vertex_deepseek_input_strategy, + vertex_mistral_input_strategy, +) + +COMMON_FIELDS: Final = frozenset( + {"contract", "model", "document", "custom_llm_provider", "vertex_project", "vertex_location"} +) +SUPPORTED_OCR_PROVIDERS: Final = frozenset({"mistral", "azure_ai", "reducto", "vertex_ai"}) +ACTIVE_OCR_MODELS: Final = frozenset( + ( + *MISTRAL_MODELS, + *AZURE_MISTRAL_MODELS, + *AZURE_DOCUMENT_INTELLIGENCE_MODELS, + *VERTEX_MISTRAL_MODELS, + *VERTEX_DEEPSEEK_MODELS, + *REDUCTO_V3_MODELS, + *REDUCTO_LEGACY_MODELS, + ) +) +_MISTRAL_2512_OR_NEWER: Final = frozenset( + { + "mistral/mistral-ocr-2512", + "mistral/mistral-ocr-3", + "mistral/mistral-ocr-3-0", + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-latest", + } +) +_MISTRAL_4_OR_NEWER: Final = frozenset( + { + "mistral/mistral-ocr-4", + "mistral/mistral-ocr-4-0", + "mistral/mistral-ocr-4-1", + "mistral/mistral-ocr-latest", + } +) +_MISTRAL_OPTION_GROUPS: Final = frozenset( + { + frozenset[str](), + *( + frozenset({field}) + for field in ( + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "include_blocks", + ) + ), + frozenset({"document_annotation_format", "document_annotation_prompt"}), + frozenset({"include_blocks", "confidence_scores_granularity"}), + } +) +_MISTRAL_2505_OPTION_GROUPS: Final = frozenset( + { + frozenset[str](), + *( + frozenset({field}) + for field in ( + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "confidence_scores_granularity", + ) + ), + frozenset({"document_annotation_format", "document_annotation_prompt"}), + } +) +_AZURE_MISTRAL_OPTION_GROUPS: Final = _MISTRAL_2505_OPTION_GROUPS - { + frozenset({"document_annotation_format", "document_annotation_prompt"}) +} +_REDUCTO_FORMATTING_INCLUDE_GROUPS: Final = ( + (), + ("hyperlinks",), + ("change_tracking", "highlight", "comments"), + ("signatures", "ignore_watermarks"), +) +_REDUCTO_FILTER_BLOCK_GROUPS: Final = ( + (), + ("Header",), + ("Header", "Footer", "Page Number"), + ("Figure", "Table", "Key Value"), +) +_REDUCTO_RETURN_IMAGE_GROUPS: Final = ( + (), + ("figure",), + ("table",), + ("page",), + ("figure", "table"), +) +_FIND_SETTINGS: Final = settings(max_examples=2_000, deadline=None, derandomize=True, database=None) +_FixtureInputT = TypeVar("_FixtureInputT") +INLINE_IMAGE_DATA_URI: Final = "data:image/png;base64,dGVzdA==" +_MapOcrParams = Callable[[dict[str, object], dict[str, object], str], dict[str, object]] +_TransformOcrRequest = Callable[ + [str, DocumentType, dict[str, object], dict[str, object]], + OCRRequestData, +] +_GetCompleteUrl = Callable[[str | None, str, dict[str, object]], str] + + +def _transform_with_stubbed_download( + transform_request: _TransformOcrRequest, + model: str, + document: DocumentType, + mapped: dict[str, object], +) -> OCRRequestData: + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + if source.startswith("data:"): + return transform_request(model, document, mapped, {}) + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + with respx.mock(assert_all_called=False) as router: + router.route(method="GET").mock( + return_value=httpx.Response(200, content=b"\x00", headers={"content-type": media_type}) + ) + return transform_request(model, document, mapped, {}) + + +def _find_fixture( + strategy: SearchStrategy[_FixtureInputT], + predicate: Callable[[_FixtureInputT], bool], +) -> _FixtureInputT: + return find(strategy, predicate, settings=_FIND_SETTINGS) + + +def _document_transport(document: ImageUrlDocument | DocumentUrlDocument) -> tuple[str, str]: + if isinstance(document, ImageUrlDocument): + source: Final = document.image_url.url if isinstance(document.image_url, ImageUrlValue) else document.image_url + return document.type, "data" if source.startswith("data:") else "remote" + return document.type, "data" if document.document_url.startswith("data:") else "remote" + + +def _normalized_azure_pages(pages: object) -> str: + if isinstance(pages, str): + return pages.replace(" ", "") + assert isinstance(pages, list) + raw_pages: Final = cast(list[object], pages) + if all(isinstance(page, int) for page in raw_pages): + integer_pages: Final = cast(list[int], raw_pages) + return ",".join(str(page + 1) for page in sorted(set(integer_pages))) + string_pages: Final = cast(list[str], raw_pages) + return ",".join(page.strip() for page in string_pages) + + +def test_structured_pdf_exercises_semantic_ocr_features() -> None: + encoded: Final = structured_pdf_data_uri().partition(",")[2] + pdf: Final = base64.b64decode(encoded, validate=True) + + assert pdf.startswith(b"%PDF-1.") + assert b"/Count 5" in pdf + assert pdf.count(b"/Subtype /Image") == 3 + assert all( + marker in pdf + for marker in ( + b"/Width 120", + b"/Width 320", + b"/Width 360", + b"/Subtype /Highlight", + b"/Subtype /Link", + b"/Subtype /Text", + b"/Title (Quarterly Operations Report)", + ) + ) + assert b"Invoice Number: INV-2048" in pdf + assert b"Formula: gross margin" in pdf + assert b"Approved by: Jordan Lee" in pdf + + +class _ModelRegistryEntry(BaseModel): + model_config = ConfigDict(frozen=True, extra="ignore") + + mode: str | None = None + litellm_provider: str | None = None + deprecation_date: date | None = None + + +MODEL_REGISTRY: Final = TypeAdapter(dict[str, dict[str, JsonValue]]) + + +def _provider_fields(model: type[OcrSdkInputBase]) -> set[str]: + return set(model.model_fields) - COMMON_FIELDS + + +def _supported_params(config: BaseOCRConfig, model: str) -> set[str]: + get_supported_params: Final = cast(Callable[[str], list[str]], config.get_supported_ocr_params) + return set(get_supported_params(model)) + + +def _mistral_input(**params: object) -> MistralOcrSdkInput: + return MistralOcrSdkInput.model_validate( + { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "image_url", "image_url": "https://example.com/image.png"}, + **params, + } + ) + + +def _reducto_document() -> ReductoDocumentUrlDocument: + return ReductoDocumentUrlDocument( + type="document_url", + document_url="reducto://fixture-document.pdf", + ) + + +def test_fixture_catalogs_match_active_registered_ocr_models() -> None: + registry_path: Final = Path(__file__).resolve().parents[6] / "model_prices_and_context_window.json" + registry: Final = MODEL_REGISTRY.validate_json(registry_path.read_text(encoding="utf-8")) + active_registered: Final = frozenset( + model + for model, raw_metadata in registry.items() + if raw_metadata.get("mode") == "ocr" and raw_metadata.get("litellm_provider") in SUPPORTED_OCR_PROVIDERS + for metadata in (_ModelRegistryEntry.model_validate(raw_metadata),) + if metadata.deprecation_date is None or metadata.deprecation_date > date.today() + ) + + assert ACTIVE_OCR_MODELS == active_registered + + +@pytest.mark.parametrize( + ("fixture_model", "provider_config", "model"), + ( + (MistralOcrSdkInput, MistralOCRConfig(), "mistral-ocr-latest"), + (AzureMistralOcrSdkInput, AzureAIOCRConfig(), "mistral-document-ai-2512"), + ( + AzureDocumentIntelligenceOcrSdkInput, + AzureDocumentIntelligenceOCRConfig(), + "doc-intelligence/prebuilt-layout", + ), + (VertexMistralOcrSdkInput, VertexAIOCRConfig(), "mistral-ocr-2505"), + (VertexDeepSeekOcrSdkInput, VertexAIDeepSeekOCRConfig(), "deepseek-ai/deepseek-ocr-maas"), + (ReductoParseV3SdkInput, ReductoParseV3Config(), "parse-v3"), + (ReductoParseLegacySdkInput, ReductoParseLegacyConfig(), "parse-legacy"), + ), +) +def test_fixture_fields_match_provider_config( + fixture_model: type[OcrSdkInputBase], provider_config: BaseOCRConfig, model: str +) -> None: + assert _provider_fields(fixture_model) == _supported_params(provider_config, model) + + +@pytest.mark.parametrize( + "sdk_input", + ( + AzureMistralOcrSdkInput( + model="azure_ai/mistral-document-ai-2512", + document=ImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + ), + VertexMistralOcrSdkInput( + document=ImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + vertex_project="project-1", + ), + AzureDocumentIntelligenceOcrSdkInput( + model="azure_ai/doc-intelligence/prebuilt-layout", + document=ImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + ), + VertexDeepSeekOcrSdkInput( + document=ImageUrlDocument(type="image_url", image_url="data:image/png;base64,AA=="), + vertex_project="project-1", + ), + ), +) +def test_provider_contract_is_explicit_but_not_forwarded(sdk_input: OcrSdkInput) -> None: + assert sdk_input.canonical_input()["contract"] == sdk_input.contract + assert "contract" not in sdk_input.as_sdk_kwargs() + + +@pytest.mark.parametrize("legacy_key", ("boundary", None)) +def test_ocr_parity_case_migrates_legacy_contract_metadata(legacy_key: str | None) -> None: + litellm_input: Final[dict[str, object]] = { + "model": "mistral/mistral-ocr-latest", + "document": {"type": "document_url", "document_url": "https://example.com/document.pdf"}, + } + if legacy_key is not None: + litellm_input[legacy_key] = "mistral" + + fixture: Final = OcrParityCase.model_validate({"litellm_input": litellm_input, "provider_responses": ()}) + + assert fixture.litellm_input.contract == "mistral" + + +def test_mistral_input_preserves_omission_and_explicit_boolean_values() -> None: + omitted: Final = _mistral_input().as_sdk_kwargs() + explicit: Final = _mistral_input(extract_header=False, include_blocks=True).as_sdk_kwargs() + + assert "extract_header" not in omitted + assert "include_blocks" not in omitted + assert explicit["extract_header"] is False + assert explicit["include_blocks"] is True + + +def test_mistral_input_supports_document_and_page_variants() -> None: + nested_image: Final = MistralOcrSdkInput( + model="mistral/mistral-ocr-4-1", + document=ImageUrlDocument( + type="image_url", + image_url=ImageUrlValue(url="https://example.com/image.png", detail="high"), + ), + pages="0,2-4", + ) + named_document: Final = MistralOcrSdkInput( + model="mistral/mistral-ocr-2512", + document=DocumentUrlDocument( + type="document_url", + document_url="https://example.com/document.pdf", + document_name="invoice.pdf", + ), + ) + + assert nested_image.canonical_input()["document"] == { + "type": "image_url", + "image_url": {"url": "https://example.com/image.png", "detail": "high"}, + } + assert nested_image.as_sdk_kwargs()["pages"] == "0,2-4" + assert named_document.canonical_input()["document"] == { + "type": "document_url", + "document_url": "https://example.com/document.pdf", + "document_name": "invoice.pdf", + } + + +def test_mistral_annotation_schema_serializes_provider_alias() -> None: + annotation: Final = JsonSchemaResponseFormat( + type="json_schema", + json_schema=JsonSchemaDefinition( + name="invoice", + schema={"type": "object"}, + ), + ) + sdk_input: Final = _mistral_input( + document_annotation_format=annotation, + document_annotation_prompt="Extract invoice fields", + ) + + assert sdk_input.canonical_input()["document_annotation_format"] == { + "type": "json_schema", + "json_schema": { + "name": "invoice", + "schema": {"type": "object"}, + }, + } + + +def test_mistral_annotation_prompt_requires_format() -> None: + with pytest.raises(ValidationError, match="requires document_annotation_format"): + _mistral_input(document_annotation_prompt="Extract invoice fields") + + +@pytest.mark.parametrize("field", ("extract_header", "extract_footer", "include_blocks")) +def test_mistral_nonnullable_booleans_reject_null(field: str) -> None: + with pytest.raises(ValidationError): + _mistral_input(**{field: None}) + + +def test_unqualified_models_require_explicit_provider() -> None: + with pytest.raises(ValidationError, match="custom_llm_provider='mistral'"): + MistralOcrSdkInput( + model="mistral-ocr-latest", + document=ImageUrlDocument(type="image_url", image_url="https://example.com/image.png"), + ) + with pytest.raises(ValidationError, match="custom_llm_provider='reducto'"): + ReductoParseV3SdkInput(model="parse-v3", document=_reducto_document()) + + +@pytest.mark.parametrize("model", tuple(model.removeprefix("mistral/") for model in MISTRAL_MODELS)) +def test_unqualified_mistral_models_accept_explicit_provider(model: str) -> None: + sdk_input: Final = MistralOcrSdkInput.model_validate( + { + "model": model, + "custom_llm_provider": "mistral", + "document": ImageUrlDocument(type="image_url", image_url="https://example.com/image.png"), + } + ) + + assert sdk_input.model == model + + +@pytest.mark.parametrize( + ("model", "model_type"), + (("parse-v3", ReductoParseV3SdkInput), ("parse-legacy", ReductoParseLegacySdkInput)), +) +def test_unqualified_reducto_models_accept_explicit_provider( + model: str, model_type: type[ReductoParseV3SdkInput] | type[ReductoParseLegacySdkInput] +) -> None: + sdk_input: Final = model_type.model_validate( + {"model": model, "custom_llm_provider": "reducto", "document": _reducto_document()} + ) + + assert sdk_input.model == model + + +@pytest.mark.parametrize( + "document", + ( + {"type": "image_url", "image_url": "data:image/png;base64,AA=="}, + {"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + ), +) +def test_vertex_deepseek_request_maps_both_document_types_to_image_content( + document: DocumentType, +) -> None: + request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request( # pyright: ignore[reportUnknownMemberType] + model="deepseek-ai/deepseek-ocr-maas", + document=document, + optional_params={}, + headers={}, + ) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + data: Final = cast(dict[str, object], request.data) + messages: Final = cast(list[dict[str, object]], data["messages"]) + content: Final = cast(list[dict[str, object]], messages[0]["content"]) + assert content == [{"type": "image_url", "image_url": document[source_key]}] + + +@pytest.mark.parametrize( + "sdk_input", + ( + ReductoParseV3SdkInput(model="reducto/parse-v3", document=_reducto_document()), + ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=_reducto_document()), + ), +) +def test_reducto_parity_cases_are_non_strict_xfails( + sdk_input: ReductoParseV3SdkInput | ReductoParseLegacySdkInput, +) -> None: + marks: Final = ocr_fixture_marks(OcrParityCase(litellm_input=sdk_input, provider_responses=())) + + assert len(marks) == 1 + assert marks[0].mark.name == "xfail" + assert marks[0].mark.kwargs["strict"] is False + + +def test_supported_parity_cases_have_no_marks() -> None: + sdk_input: Final = _mistral_input() + + assert ocr_fixture_marks(OcrParityCase(litellm_input=sdk_input, provider_responses=())) == () + + +def test_reducto_v3_preserves_nested_provider_params() -> None: + sdk_input: Final = ReductoParseV3SdkInput( + model="reducto/parse-v3", + document=_reducto_document(), + formatting=ReductoFormatting(table_output_format="html", include=["hyperlinks"]), + retrieval=ReductoRetrieval(chunking=ReductoChunking(chunk_mode="variable", chunk_size=250, chunk_overlap=32)), + settings=ReductoSettings(embed_pdf_metadata=True, embed_pdf_metadata_dpi=250, page_range=[1, 3]), + ) + + assert sdk_input.as_sdk_kwargs()["formatting"] == { + "table_output_format": "html", + "include": ["hyperlinks"], + } + assert sdk_input.as_sdk_kwargs()["retrieval"] == { + "chunking": {"chunk_mode": "variable", "chunk_size": 250, "chunk_overlap": 32} + } + assert sdk_input.as_sdk_kwargs()["settings"] == { + "embed_pdf_metadata": True, + "embed_pdf_metadata_dpi": 250, + "page_range": [1, 3], + } + + +def test_reducto_optional_objects_reject_explicit_null() -> None: + with pytest.raises(ValidationError): + ReductoParseV3SdkInput.model_validate( + { + "model": "reducto/parse-v3", + "document": _reducto_document(), + "formatting": None, + } + ) + + +@pytest.mark.parametrize( + "source", + ( + "https://example.com/document.pdf", + "not-a-document", + "data:application/pdf,not-base64", + "data:application/pdf;base64,not!base64", + ), +) +def test_reducto_document_rejects_unsupported_sources(source: str) -> None: + with pytest.raises(ValidationError): + ReductoDocumentUrlDocument(type="document_url", document_url=source) + + +def test_reducto_nested_constraints() -> None: + with pytest.raises(ValidationError, match="less than chunk_size"): + ReductoChunking(chunk_mode="variable", chunk_size=100, chunk_overlap=100) + with pytest.raises(ValidationError, match="greater than or equal to start"): + ReductoPageRange(start=3, end=2) + with pytest.raises(ValidationError): + ReductoSettings(embed_pdf_metadata_dpi=49) + with pytest.raises(ValidationError, match="must be unique"): + ReductoFormatting(include=["hyperlinks", "hyperlinks"]) + + +@settings(max_examples=100, deadline=None) +@given(model=st.sampled_from(MISTRAL_MODELS), data=st.data()) +def test_mistral_strategy_only_generates_bounded_valid_sdk_inputs(model: str, data: DataObject) -> None: + sdk_input: Final = data.draw(mistral_input_strategy(model, INLINE_IMAGE_DATA_URI)) + assert MistralOcrSdkInput.model_validate(sdk_input.canonical_input()) == sdk_input + optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} + assert optional_fields in _MISTRAL_OPTION_GROUPS + if sdk_input.pages is not None: + assert sdk_input.pages in ([0], [0, 1], "0-2") + if sdk_input.image_limit is not None: + assert sdk_input.image_limit == 1 + if sdk_input.image_min_size is not None: + assert sdk_input.image_min_size == 300 + if sdk_input.table_format is not None: + assert sdk_input.table_format in {"markdown", "html"} + if sdk_input.confidence_scores_granularity is not None: + assert sdk_input.confidence_scores_granularity in {"page", "word", "block"} + if sdk_input.confidence_scores_granularity == "block": + assert sdk_input.include_blocks is True + if model not in _MISTRAL_2512_OR_NEWER: + assert optional_fields.isdisjoint({"extract_header", "extract_footer", "table_format"}) + if model not in _MISTRAL_4_OR_NEWER: + assert "include_blocks" not in optional_fields + assert not isinstance(sdk_input.pages, str) + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + "transport", + ( + ("image_url", "remote"), + ("image_url", "data"), + ("document_url", "remote"), + ("document_url", "data"), + ), +) +def test_mistral_strategy_reaches_every_document_transform_branch(transport: tuple[str, str]) -> None: + sdk_input: Final = _find_fixture( + mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI), + lambda candidate: _document_transport(candidate.document) == transport, + ) + + assert _document_transport(sdk_input.document) == transport + + +@settings(max_examples=100, deadline=None) +@given(sdk_input=mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI)) +def test_mistral_strategy_values_survive_the_request_transform(sdk_input: MistralOcrSdkInput) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = MistralOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + request: Final = transform_request(model, document, mapped, {}) + request_data: Final = cast(dict[str, object], request.data) + + assert mapped == optional_params + assert request_data == {"model": model, "document": document, **optional_params} + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [0, 1]), + ("pages", "0-2"), + ("include_image_base64", False), + ("include_image_base64", True), + ("image_limit", 1), + ("image_min_size", 300), + ("extract_header", False), + ("extract_header", True), + ("extract_footer", False), + ("extract_footer", True), + ("table_format", "markdown"), + ("table_format", "html"), + ("confidence_scores_granularity", "page"), + ("confidence_scores_granularity", "word"), + ("confidence_scores_granularity", "block"), + ("include_blocks", False), + ("include_blocks", True), + ), +) +def test_mistral_strategy_reaches_every_finite_scalar_value(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_reducto_v3_strategy_only_generates_bounded_valid_sdk_inputs(sdk_input: ReductoParseV3SdkInput) -> None: + assert ReductoParseV3SdkInput.model_validate(sdk_input.canonical_input()) == sdk_input + option_groups: Final = frozenset(sdk_input.model_fields_set) & {"formatting", "retrieval", "settings"} + assert len(option_groups) <= 1 + if "formatting" in option_groups: + formatting_fields: Final = frozenset(sdk_input.formatting.model_fields_set) + assert len(formatting_fields) == 1 + if "table_output_format" in formatting_fields: + assert sdk_input.formatting.table_output_format in {"dynamic", "html", "md", "json", "csv", "jsonbbox"} + if "add_page_markers" in formatting_fields: + assert sdk_input.formatting.add_page_markers in {False, True} + if "merge_tables" in formatting_fields: + assert sdk_input.formatting.merge_tables in {False, True} + if "include" in formatting_fields: + assert tuple(sdk_input.formatting.include) in _REDUCTO_FORMATTING_INCLUDE_GROUPS + if "retrieval" in option_groups: + retrieval_fields: Final = frozenset(sdk_input.retrieval.model_fields_set) + assert retrieval_fields in { + frozenset({"chunking"}), + frozenset({"filter_blocks"}), + frozenset({"chunking", "embedding_optimized"}), + } + chunking: Final = sdk_input.retrieval.chunking + if "chunking" in retrieval_fields: + assert chunking.chunk_mode in {"variable", "section", "page", "disabled", "block", "page_sections"} + assert chunking.chunk_size in {None, 250, 1000, 1500} + assert chunking.chunk_overlap in {0, 32, 128} + if chunking.chunk_size is not None or chunking.chunk_overlap: + assert chunking.chunk_mode == "variable" + if chunking.chunk_overlap: + assert chunking.chunk_size == 1000 + if "filter_blocks" in retrieval_fields: + assert tuple(sdk_input.retrieval.filter_blocks) in _REDUCTO_FILTER_BLOCK_GROUPS + if "embedding_optimized" in retrieval_fields: + assert chunking.chunk_mode == "variable" + assert chunking.chunk_size is None + assert chunking.chunk_overlap == 0 + assert sdk_input.retrieval.embedding_optimized in {False, True} + if "settings" in option_groups: + settings_fields: Final = frozenset(sdk_input.settings.model_fields_set) + assert settings_fields in { + frozenset({"model"}), + frozenset({"ocr_system"}), + frozenset({"extraction_mode"}), + frozenset({"return_ocr_data"}), + frozenset({"return_images"}), + frozenset({"embed_pdf_metadata"}), + frozenset({"embed_pdf_metadata", "embed_pdf_metadata_dpi"}), + frozenset({"timeout"}), + frozenset({"page_range"}), + } + assert settings_fields.isdisjoint( + { + "force_url_result", + "force_file_extension", + "persist_results", + "tenant_throttling", + "document_password", + "hybrid_vpc", + } + ) + if "model" in settings_fields: + assert sdk_input.settings.model == "r-1" + if "ocr_system" in settings_fields: + assert sdk_input.settings.ocr_system in {"standard", "legacy"} + if "extraction_mode" in settings_fields: + assert sdk_input.settings.extraction_mode in {"hybrid", "ocr", "metadata"} + if "return_ocr_data" in settings_fields: + assert sdk_input.settings.return_ocr_data is True + if "return_images" in settings_fields: + assert tuple(sdk_input.settings.return_images) in _REDUCTO_RETURN_IMAGE_GROUPS + if "embed_pdf_metadata_dpi" in settings_fields: + assert sdk_input.settings.embed_pdf_metadata is True + assert sdk_input.settings.embed_pdf_metadata_dpi in {50, 100, 250} + if "timeout" in settings_fields: + assert sdk_input.settings.timeout == 300.0 + if sdk_input.settings.page_range is not None: + dumped_range: Final = cast( + dict[str, object], sdk_input.settings.model_dump(mode="json", exclude_unset=True) + )["page_range"] + assert dumped_range in ( + {"start": 1, "end": 1}, + {"start": 1, "end": 3}, + [{"start": 1, "end": 2}, {"start": 4, "end": 5}], + ) + + +@settings(max_examples=60, deadline=None) +@given(sdk_input=reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_reducto_v3_strategy_values_survive_the_request_transform(sdk_input: ReductoParseV3SdkInput) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = { + name: value for name, value in sdk_kwargs.items() if name not in {"model", "document", "custom_llm_provider"} + } + config: Final = ReductoParseV3Config() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + with patch.object(config, "_ensure_file_id_sync", return_value="reducto://fixture-document.pdf"): + request: Final = transform_request(model, document, mapped, {}) + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "input": "reducto://fixture-document.pdf", + **optional_params, + } + + +def test_reducto_v3_strategy_reaches_image_upload_branch_without_options() -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: isinstance(candidate.document, ReductoImageUrlDocument), + ) + + assert isinstance(sdk_input.document, ReductoImageUrlDocument) + assert sdk_input.document.image_url.startswith("data:image/") + assert sdk_input.model_fields_set == {"model", "document"} + + +@pytest.mark.parametrize( + ("model", "provider"), + (("reducto/parse-v3", None), ("parse-v3", "reducto")), +) +def test_reducto_v3_strategy_reaches_every_routing_form(model: str, provider: str | None) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: candidate.model == model and candidate.custom_llm_provider == provider, + ) + + assert sdk_input.model == model + assert sdk_input.custom_llm_provider == provider + + +@pytest.mark.parametrize("table_format", ("dynamic", "html", "md", "json", "csv", "jsonbbox")) +def test_reducto_v3_strategy_reaches_every_table_format(table_format: str) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "formatting" in candidate.model_fields_set + and "table_output_format" in candidate.formatting.model_fields_set + and candidate.formatting.table_output_format == table_format + ), + ) + + assert sdk_input.formatting.table_output_format == table_format + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("add_page_markers", False), + ("add_page_markers", True), + ("merge_tables", False), + ("merge_tables", True), + ), +) +def test_reducto_v3_strategy_reaches_every_formatting_boolean(field: str, value: bool) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "formatting" in candidate.model_fields_set + and field in candidate.formatting.model_fields_set + and getattr(candidate.formatting, field) is value + ), + ) + + assert getattr(sdk_input.formatting, field) is value + + +@pytest.mark.parametrize("include", _REDUCTO_FORMATTING_INCLUDE_GROUPS) +def test_reducto_v3_strategy_reaches_every_formatting_include(include: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "formatting" in candidate.model_fields_set + and "include" in candidate.formatting.model_fields_set + and tuple(candidate.formatting.include) == include + ), + ) + + assert tuple(sdk_input.formatting.include) == include + + +@pytest.mark.parametrize("chunk_mode", ("variable", "section", "page", "disabled", "block", "page_sections")) +def test_reducto_v3_strategy_reaches_every_chunk_mode(chunk_mode: str) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "retrieval" in candidate.model_fields_set + and "chunking" in candidate.retrieval.model_fields_set + and candidate.retrieval.chunking.chunk_mode == chunk_mode + ), + ) + + assert sdk_input.retrieval.chunking.chunk_mode == chunk_mode + + +@pytest.mark.parametrize("chunk_size", (250, 1000, 1500)) +def test_reducto_v3_strategy_reaches_every_chunk_size(chunk_size: int) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: candidate.retrieval.chunking.chunk_size == chunk_size, + ) + + assert sdk_input.retrieval.chunking.chunk_mode == "variable" + assert sdk_input.retrieval.chunking.chunk_size == chunk_size + + +@pytest.mark.parametrize("chunk_overlap", (32, 128)) +def test_reducto_v3_strategy_reaches_every_chunk_overlap(chunk_overlap: int) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: candidate.retrieval.chunking.chunk_overlap == chunk_overlap, + ) + + assert sdk_input.retrieval.chunking.chunk_mode == "variable" + assert sdk_input.retrieval.chunking.chunk_size == 1000 + assert sdk_input.retrieval.chunking.chunk_overlap == chunk_overlap + + +@pytest.mark.parametrize("filter_blocks", _REDUCTO_FILTER_BLOCK_GROUPS) +def test_reducto_v3_strategy_reaches_every_filter_block_group(filter_blocks: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "retrieval" in candidate.model_fields_set + and "filter_blocks" in candidate.retrieval.model_fields_set + and tuple(candidate.retrieval.filter_blocks) == filter_blocks + ), + ) + + assert tuple(sdk_input.retrieval.filter_blocks) == filter_blocks + + +@pytest.mark.parametrize("embedding_optimized", (False, True)) +def test_reducto_v3_strategy_reaches_every_embedding_setting(embedding_optimized: bool) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "retrieval" in candidate.model_fields_set + and "embedding_optimized" in candidate.retrieval.model_fields_set + and candidate.retrieval.embedding_optimized is embedding_optimized + ), + ) + + assert sdk_input.retrieval.chunking.chunk_mode == "variable" + assert sdk_input.retrieval.embedding_optimized is embedding_optimized + + +@pytest.mark.parametrize("dpi", (50, 100, 250)) +def test_reducto_v3_strategy_reaches_every_metadata_dpi(dpi: int) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set + and "embed_pdf_metadata_dpi" in candidate.settings.model_fields_set + and candidate.settings.embed_pdf_metadata_dpi == dpi + ), + ) + + assert sdk_input.settings.embed_pdf_metadata is True + assert sdk_input.settings.embed_pdf_metadata_dpi == dpi + + +def test_reducto_v3_strategy_reaches_metadata_with_default_dpi_omitted() -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set and candidate.settings.model_fields_set == {"embed_pdf_metadata"} + ), + ) + + assert sdk_input.settings.embed_pdf_metadata is True + assert "embed_pdf_metadata_dpi" not in sdk_input.settings.model_fields_set + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("model", "r-1"), + ("ocr_system", "standard"), + ("ocr_system", "legacy"), + ("extraction_mode", "hybrid"), + ("extraction_mode", "ocr"), + ("extraction_mode", "metadata"), + ("return_ocr_data", True), + ("timeout", 300.0), + ), +) +def test_reducto_v3_strategy_reaches_every_scalar_setting(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set + and field in candidate.settings.model_fields_set + and getattr(candidate.settings, field) == value + ), + ) + + assert getattr(sdk_input.settings, field) == value + + +@pytest.mark.parametrize("return_images", _REDUCTO_RETURN_IMAGE_GROUPS) +def test_reducto_v3_strategy_reaches_every_return_image_group(return_images: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set + and "return_images" in candidate.settings.model_fields_set + and tuple(candidate.settings.return_images) == return_images + ), + ) + + assert tuple(sdk_input.settings.return_images) == return_images + + +@pytest.mark.parametrize( + "page_range", + ( + {"start": 1, "end": 1}, + {"start": 1, "end": 3}, + [{"start": 1, "end": 2}, {"start": 4, "end": 5}], + ), +) +def test_reducto_v3_strategy_reaches_every_page_range_shape(page_range: object) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + cast( + dict[str, object], + candidate.settings.model_dump(mode="json", exclude_unset=True), + ).get("page_range") + == page_range + ), + ) + + assert sdk_input.settings.model_dump(mode="json", exclude_unset=True)["page_range"] == page_range + + +@settings(max_examples=10, deadline=None) +@given(sdk_input=reducto_legacy_input_strategy()) +def test_reducto_legacy_strategy_generates_valid_litellm_inputs(sdk_input: ReductoParseLegacySdkInput) -> None: + assert ReductoParseLegacySdkInput.model_validate(sdk_input.canonical_input()) == sdk_input + assert "enhance" not in sdk_input.model_fields_set + + +@settings(max_examples=10, deadline=None) +@given(sdk_input=reducto_legacy_input_strategy()) +def test_reducto_legacy_strategy_values_survive_the_request_transform( + sdk_input: ReductoParseLegacySdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + config: Final = ReductoParseLegacyConfig() + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + + with patch.object(config, "_ensure_file_id_sync", return_value="reducto://fixture-document.pdf"): + request: Final = transform_request(model, document, {}, {}) + + assert cast(dict[str, object], request.data) == { + "document_url": "reducto://fixture-document.pdf", + } + + +@pytest.mark.parametrize( + ("model", "provider"), + (("reducto/parse-legacy", None), ("parse-legacy", "reducto")), +) +def test_reducto_legacy_strategy_reaches_every_routing_form(model: str, provider: str | None) -> None: + sdk_input: Final = _find_fixture( + reducto_legacy_input_strategy(), + lambda candidate: candidate.model == model and candidate.custom_llm_provider == provider, + ) + + assert sdk_input.model == model + assert sdk_input.custom_llm_provider == provider + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_azure_mistral_strategy_is_contained_to_gateway_capabilities( + sdk_input: AzureMistralOcrSdkInput, +) -> None: + optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} + + assert optional_fields in _AZURE_MISTRAL_OPTION_GROUPS + assert optional_fields.isdisjoint( + { + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "include_blocks", + "id", + } + ) + assert not isinstance(sdk_input.pages, str) + assert sdk_input.confidence_scores_granularity in {None, "page", "word"} + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [0, 1]), + ("include_image_base64", False), + ("include_image_base64", True), + ("image_limit", 1), + ("image_min_size", 300), + ("confidence_scores_granularity", "page"), + ("confidence_scores_granularity", "word"), + ), +) +def test_azure_mistral_strategy_reaches_every_gateway_scalar(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +@pytest.mark.parametrize("field", ("bbox_annotation_format", "document_annotation_format")) +def test_azure_mistral_strategy_reaches_every_gateway_schema(field: str) -> None: + sdk_input: Final = _find_fixture( + azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: frozenset(candidate.model_fields_set) - {"model", "document"} == frozenset({field}), + ) + + assert frozenset(sdk_input.model_fields_set) - {"model", "document"} == {field} + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_azure_mistral_strategy_exercises_url_conversion_and_inline_bypass( + sdk_input: AzureMistralOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = AzureAIOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + request: Final = _transform_with_stubbed_download(transform_request, model, document, mapped) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + expected_document: Final = dict(document) + if not source.startswith("data:"): + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + expected_document[source_key] = f"data:{media_type};base64,AA==" + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "model": model, + "document": expected_document, + **optional_params, + } + + +@settings(max_examples=30, deadline=None) +@given(sdk_input=azure_document_intelligence_input_strategy()) +def test_azure_document_intelligence_strategy_only_generates_litellm_inputs( + sdk_input: AzureDocumentIntelligenceOcrSdkInput, +) -> None: + assert sdk_input.req_format == "litellm" + assert sdk_input.model in AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS + assert "contract" not in sdk_input.as_sdk_kwargs() + optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} + assert optional_fields in { + frozenset[str](), + frozenset({"pages"}), + frozenset({"features"}), + frozenset({"pages", "features"}), + frozenset({"req_format"}), + } + if sdk_input.pages is not None: + assert sdk_input.pages in ([0], [2, 0, 0, 1], ["1", "2-4"], "1-4, 5", [0, 1]) + if isinstance(sdk_input.features, list): + assert tuple(sdk_input.features) in { + ("languages",), + ("ocrHighResolution",), + ("barcodes",), + ("formulas",), + ("styleFont",), + ("keyValuePairs",), + ("languages", "styleFont"), + } + if isinstance(sdk_input.features, str): + assert sdk_input.features == "languages, styleFont" + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_document_intelligence_input_strategy()) +def test_azure_document_intelligence_strategy_exercises_request_transform( + sdk_input: AzureDocumentIntelligenceOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = AzureDocumentIntelligenceOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + get_complete_url: Final = cast(_GetCompleteUrl, config.get_complete_url) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + url: Final = get_complete_url("https://document.example", model, mapped) + query: Final = parse_qs(urlparse(url).query) + request: Final = transform_request(model, document, mapped, {}) + + if sdk_input.pages is None: + assert "pages" not in mapped + assert "pages" not in query + else: + expected_pages: Final = _normalized_azure_pages(sdk_input.pages) + assert mapped["pages"] == expected_pages + assert query["pages"] == [expected_pages] + if sdk_input.features is None: + assert "features" not in mapped + assert "features" not in query + else: + raw_features: Final = ( + sdk_input.features.split(",") if isinstance(sdk_input.features, str) else sdk_input.features + ) + expected_features: Final = ",".join(feature.strip() for feature in raw_features) + assert mapped["features"] == expected_features + assert query["features"] == [expected_features] + + source: Final = document["document_url"] if document["type"] == "document_url" else document["image_url"] + assert isinstance(source, str) + expected_body: Final = ( + {"base64Source": source.partition(",")[2]} if source.startswith("data:") else {"urlSource": source} + ) + assert cast(dict[str, object], request.data) == expected_body + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [2, 0, 0, 1]), + ("pages", ["1", "2-4"]), + ("pages", "1-4, 5"), + ("features", ["languages"]), + ("features", ["ocrHighResolution"]), + ("features", ["barcodes"]), + ("features", ["formulas"]), + ("features", ["styleFont"]), + ("features", ["keyValuePairs"]), + ("features", "languages, styleFont"), + ), +) +def test_azure_document_intelligence_strategy_reaches_every_finite_value(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + azure_document_intelligence_input_strategy(), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +def test_azure_document_intelligence_strategy_reaches_combined_query_branch() -> None: + sdk_input: Final = _find_fixture( + azure_document_intelligence_input_strategy(), + lambda candidate: {"pages", "features"}.issubset(candidate.model_fields_set), + ) + + assert sdk_input.pages == [0, 1] + assert sdk_input.features == ["languages", "styleFont"] + + +@pytest.mark.parametrize( + "transport", + (("document_url", "data"), ("image_url", "remote")), +) +def test_azure_document_intelligence_strategy_reaches_body_source_branches( + transport: tuple[str, str], +) -> None: + sdk_input: Final = _find_fixture( + azure_document_intelligence_input_strategy(), + lambda candidate: _document_transport(candidate.document) == transport, + ) + + assert _document_transport(sdk_input.document) == transport + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) +def test_vertex_mistral_strategy_is_contained_to_2505_capabilities( + sdk_input: VertexMistralOcrSdkInput, +) -> None: + optional_fields: Final = frozenset(sdk_input.model_fields_set) - { + "model", + "document", + "vertex_project", + "vertex_location", + } + + assert optional_fields in _MISTRAL_2505_OPTION_GROUPS + assert optional_fields.isdisjoint({"extract_header", "extract_footer", "table_format", "include_blocks", "id"}) + assert not isinstance(sdk_input.pages, str) + assert sdk_input.confidence_scores_granularity in {None, "page", "word"} + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [0, 1]), + ("include_image_base64", False), + ("include_image_base64", True), + ("image_limit", 1), + ("image_min_size", 300), + ("confidence_scores_granularity", "page"), + ("confidence_scores_granularity", "word"), + ), +) +def test_vertex_mistral_strategy_reaches_every_2505_scalar(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +@pytest.mark.parametrize( + "fields", + ( + frozenset({"bbox_annotation_format"}), + frozenset({"document_annotation_format"}), + frozenset({"document_annotation_format", "document_annotation_prompt"}), + ), +) +def test_vertex_mistral_strategy_reaches_every_2505_schema_group(fields: frozenset[str]) -> None: + sdk_input: Final = _find_fixture( + vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: ( + frozenset(candidate.model_fields_set) - {"model", "document", "vertex_project", "vertex_location"} == fields + ), + ) + + assert frozenset(sdk_input.model_fields_set) - {"model", "document", "vertex_project", "vertex_location"} == fields + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) +def test_vertex_mistral_strategy_exercises_url_conversion_and_inline_bypass( + sdk_input: VertexMistralOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = { + name: value + for name, value in sdk_kwargs.items() + if name not in {"model", "document", "vertex_project", "vertex_location"} + } + config: Final = VertexAIOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + request: Final = _transform_with_stubbed_download(transform_request, model, document, mapped) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + expected_document: Final = dict(document) + if not source.startswith("data:"): + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + expected_document[source_key] = f"data:{media_type};base64,AA==" + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "model": model, + "document": expected_document, + **optional_params, + } + + +@settings(max_examples=30, deadline=None) +@given(sdk_input=vertex_deepseek_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) +def test_vertex_deepseek_strategy_only_generates_litellm_inputs( + sdk_input: VertexDeepSeekOcrSdkInput, +) -> None: + assert sdk_input.vertex_project == "project-1" + assert "contract" not in sdk_input.as_sdk_kwargs() + assert _document_transport(sdk_input.document) == ("image_url", "data") + + +def test_vertex_deepseek_strategy_reaches_documented_image_branch() -> None: + sdk_input: Final = _find_fixture( + vertex_deepseek_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: _document_transport(candidate.document) == ("image_url", "data"), + ) + + assert _document_transport(sdk_input.document) == ("image_url", "data") diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_store.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_store.py new file mode 100644 index 00000000000..2d907f4f3b8 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_fixture_store.py @@ -0,0 +1,85 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from queue import Queue +from typing import Final, Protocol, cast + +import pytest + +from .....shared.parity.fixtures.cassette import deserialize_cassette +from .....shared.parity.fixtures.pytest_support import parametrize_recorded_fixtures +from .....shared.parity.fixtures.store import FixtureEnvelope, read_fixture, recorded_fixtures +from .conftest import ocr_fixture_id, ocr_fixture_marks +from .fixtures.migrate import migrate_fixture +from .fixtures.models import OcrParityCase + + +class _Parameter(Protocol): + values: tuple[OcrParityCase, ...] + marks: tuple[pytest.Mark, ...] + + +@dataclass(frozen=True, slots=True) +class _MetafuncSpy: + fixturenames: tuple[str, ...] + calls: Queue[tuple[object, ...]] + + def parametrize(self, *args: object, **_kwargs: object) -> None: + self.calls.put(args) + + +def test_recorded_fixture_parametrization_applies_case_specific_marks() -> None: + calls: Final[Queue[tuple[object, ...]]] = Queue() + metafunc: Final = _MetafuncSpy(fixturenames=("ocr_fixture",), calls=calls) + + parametrize_recorded_fixtures( + cast(pytest.Metafunc, metafunc), + fixture_name="ocr_fixture", + case_type=OcrParityCase, + env_var="UNCONFIGURED_OCR_FIXTURE_TEST_DIRECTORY", + default_directory=Path(__file__).with_name("fixtures") / "data", + regeneration_command="unused", + id_builder=ocr_fixture_id, + marks_builder=ocr_fixture_marks, + ) + + parameters: Final = cast(tuple[_Parameter, ...], calls.get_nowait()[1]) + reducto_parameters: Final = tuple( + parameter + for parameter in parameters + if parameter.values[0].litellm_input.contract in {"reducto_v3", "reducto_legacy"} + ) + supported_parameters: Final = tuple(parameter for parameter in parameters if parameter not in reducto_parameters) + + assert reducto_parameters + assert supported_parameters + assert all(len(parameter.marks) == 1 for parameter in reducto_parameters) + assert all(parameter.marks[0].name == "xfail" for parameter in reducto_parameters) + assert all(parameter.marks[0].kwargs["strict"] is False for parameter in reducto_parameters) + assert all(parameter.marks == () for parameter in supported_parameters) + + +def test_legacy_fixture_migration_preserves_responses_and_labels_reconstructed_requests(tmp_path: Path) -> None: + case: Final = recorded_fixtures(Path(__file__).with_name("fixtures") / "data" / "mistral-ocr", OcrParityCase)[0] + timestamp: Final = datetime(2020, 1, 1, tzinfo=timezone.utc) + envelope: Final = FixtureEnvelope( + schema_version=1, + recorded_at=timestamp, + case=case.model_dump(mode="json", exclude_unset=True), + ) + legacy_path: Final = tmp_path / "legacy.json" + legacy_path.write_text(envelope.model_dump_json()) + + destination: Final = migrate_fixture(legacy_path) + + assert not legacy_path.exists() + assert read_fixture(destination, OcrParityCase) == case + cassette: Final = deserialize_cassette(destination.read_text()) + assert cassette.recorded_at == timestamp + assert cassette.parity.request_source == "python_replay" + assert len(cassette.interactions) == len(case.provider_responses) + assert cassette.interactions[0].request.method == "POST" + assert cassette.interactions[0].request.uri == "http://parity-provider.invalid/v1/ocr" + assert "authorization" not in cassette.interactions[0].request.headers diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_record_fixtures.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_record_fixtures.py new file mode 100644 index 00000000000..32dd3629635 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_record_fixtures.py @@ -0,0 +1,366 @@ +from __future__ import annotations + +import queue +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from typing import Final, cast + +import pytest +from hypothesis import find, settings +from hypothesis.strategies import SearchStrategy + +from .....shared.parity.fixtures.cli import parse_recording_args +from .....shared.parity.fixtures.inputs import generate_case_inputs +from .....shared.parity.fixtures.media import structured_pdf_data_uri +from .fixtures.azure import ( + AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, + AZURE_MISTRAL_MODELS, + AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, +) +from .fixtures.base import OcrSdkInputBase +from .fixtures.common import OcrFixtureClient, OcrRecordingTarget +from .fixtures.mistral import MISTRAL_MODELS, MISTRAL_PROVIDER_REJECTED_INPUTS +from .fixtures.record import ( + discover_targets as discover_targets_with_media, +) +from .fixtures.record import ( + require_targets, +) +from .fixtures.reducto import ( + REDUCTO_LEGACY_MODELS, + REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, + REDUCTO_V3_MODELS, + REDUCTO_V3_PROVIDER_REJECTED_INPUTS, +) +from .fixtures.vertex import ( + VERTEX_DEEPSEEK_MODELS, + VERTEX_MISTRAL_MODELS, + vertex_deepseek_provider_rejected_inputs, + vertex_mistral_provider_rejected_inputs, +) + + +class _UnusedOcrClient: + def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: + raise AssertionError(f"unexpected SDK call to {api_base} with {api_key!r} and {case_input!r}") + + +@dataclass(frozen=True, slots=True) +class _RecordingOcrClient: + calls: queue.SimpleQueue[dict[str, object]] + + def execute(self, api_base: str, api_key: str, case_input: OcrSdkInputBase) -> None: + self.calls.put({"api_base": api_base, "api_key": api_key, **case_input.as_sdk_kwargs()}) + + +_UNUSED_OCR_CLIENT: Final = _UnusedOcrClient() +_MISTRAL_PARAMS: Final = frozenset( + { + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "confidence_scores_granularity", + "include_blocks", + } +) +_MISTRAL_2512_PARAMS: Final = _MISTRAL_PARAMS - {"include_blocks"} +_MISTRAL_2505_PARAMS: Final = _MISTRAL_2512_PARAMS - {"extract_header", "extract_footer", "table_format"} +_AZURE_MISTRAL_PARAMS: Final = _MISTRAL_2505_PARAMS - {"document_annotation_prompt"} +_FIND_SETTINGS: Final = settings(max_examples=2_000, deadline=None, derandomize=True, database=None) +_INLINE_IMAGE_DATA_URI: Final = "data:image/png;base64,dGVzdA==" + + +def discover_targets(environ: Mapping[str, str], client: OcrFixtureClient) -> tuple[OcrRecordingTarget, ...]: + return discover_targets_with_media(environ, client, _INLINE_IMAGE_DATA_URI) + + +def _model(case_input: OcrSdkInputBase) -> str: + model: Final = case_input.canonical_input().get("model") + assert isinstance(model, str) + return model + + +def _find_input( + strategy: SearchStrategy[OcrSdkInputBase], + predicate: Callable[[OcrSdkInputBase], bool], +) -> OcrSdkInputBase: + return find(strategy, predicate, settings=_FIND_SETTINGS) + + +def _document_transport(case_input: OcrSdkInputBase) -> tuple[str, str]: + document: Final = cast(dict[str, object], case_input.canonical_input()["document"]) + document_type: Final = cast(str, document["type"]) + source: Final = document["image_url"] if document_type == "image_url" else document["document_url"] + assert isinstance(source, str) + return document_type, "data" if source.startswith("data:") else "remote" + + +def test_parse_args_has_no_model_selection() -> None: + args: Final = parse_recording_args(["--examples", "2", "--concurrency", "3", "--fixture-dir", "/tmp/ocr"]) + + assert args.examples == 2 + assert args.concurrency == 3 + assert args.fixture_dir == Path("/tmp/ocr") + with pytest.raises(SystemExit): + parse_recording_args(["--model", "mistral/mistral-ocr-latest"]) + + +@pytest.mark.parametrize( + "environ", + ( + {}, + {"MISTRAL_API_KEY": ""}, + {"LITELLM_API_KEY": "generic-key"}, + ), +) +def test_discovery_requires_provider_specific_key(environ: dict[str, str]) -> None: + assert discover_targets(environ, _UNUSED_OCR_CLIENT) == () + + +def test_no_discovered_targets_has_actionable_error() -> None: + with pytest.raises(SystemExit, match="supported provider API key"): + require_targets(()) + + +def test_discovery_is_explicit_per_available_provider_boundary() -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "REDUCTO_API_KEY": "reducto-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "AZURE_DOCUMENT_INTELLIGENCE_API_KEY": "document-secret", + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": "https://document.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + + assert tuple(target.name for target in targets) == ( + "mistral-ocr", + "azure-mistral", + "azure-document-intelligence", + "vertex-mistral", + "vertex-deepseek", + "reducto-v3", + "reducto-legacy", + ) + assert all("secret" not in repr(target) for target in targets) + + +def test_azure_mistral_discovery_enumerates_registered_models() -> None: + environ: Final = { + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + } + target: Final = discover_targets(environ, _UNUSED_OCR_CLIENT)[0] + + for model in AZURE_MISTRAL_MODELS: + assert ( + _model( + _find_input( + target.strategy, + lambda case_input, expected_model=model: _model(case_input) == expected_model, + ) + ) + == model + ) + + +@pytest.mark.parametrize( + ("configured", "expected"), + ( + (None, "https://api.mistral.ai"), + ("https://mistral.example/v1", "https://mistral.example"), + ("https://mistral.example/", "https://mistral.example"), + ), +) +def test_mistral_target_uses_canonical_model_and_normalized_base( + configured: str | None, + expected: str, +) -> None: + environ: Final = { + "MISTRAL_API_KEY": "mistral-secret", + **({"MISTRAL_API_BASE": configured} if configured is not None else {}), + } + targets: Final = discover_targets(environ, _UNUSED_OCR_CLIENT) + + assert len(targets) == 1 + target: Final = targets[0] + assert target.name == "mistral-ocr" + assert target.upstream.base_url == expected + assert "mistral-secret" not in repr(target) + case_inputs: Final = generate_case_inputs(target.strategy, examples=1) + assert len(case_inputs) == 1 + assert case_inputs[0].canonical_input()["model"] in MISTRAL_MODELS + + +def test_mistral_target_invocation_forwards_discovered_credentials() -> None: + calls: Final[queue.SimpleQueue[dict[str, object]]] = queue.SimpleQueue() + + client: Final = _RecordingOcrClient(calls) + target: Final = discover_targets({"MISTRAL_API_KEY": "mistral-secret"}, client)[0] + case_input: Final = generate_case_inputs(target.strategy, examples=1)[0] + + target.invocation.execute("http://127.0.0.1:1234", case_input) + + kwargs: Final = calls.get_nowait() + assert kwargs["api_base"] == "http://127.0.0.1:1234" + assert kwargs["api_key"] == "mistral-secret" + assert kwargs["model"] in MISTRAL_MODELS + + +def test_every_target_strategy_reaches_every_recording_model_and_coverage_param() -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "REDUCTO_API_KEY": "reducto-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "AZURE_DOCUMENT_INTELLIGENCE_API_KEY": "document-secret", + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": "https://document.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + expected: Final[dict[str, tuple[tuple[str, ...], frozenset[str]]]] = { + "mistral-ocr": (MISTRAL_MODELS, _MISTRAL_PARAMS), + "azure-mistral": (AZURE_MISTRAL_MODELS, _AZURE_MISTRAL_PARAMS), + "azure-document-intelligence": ( + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, + frozenset({"pages", "features", "req_format"}), + ), + "vertex-mistral": (VERTEX_MISTRAL_MODELS, _MISTRAL_2505_PARAMS), + "vertex-deepseek": (VERTEX_DEEPSEEK_MODELS, frozenset[str]()), + "reducto-v3": (REDUCTO_V3_MODELS, frozenset({"formatting", "retrieval", "settings"})), + "reducto-legacy": (REDUCTO_LEGACY_MODELS, frozenset[str]()), + } + + for target in targets: + expected_models, expected_params = expected[target.name] + for model in expected_models: + assert ( + _model( + _find_input( + target.strategy, + lambda case_input, expected_model=model: _model(case_input) == expected_model, + ) + ) + == model + ) + for param in expected_params: + reached = _find_input( + target.strategy, + lambda case_input, expected_param=param: expected_param in case_input.as_sdk_kwargs(), + ) + assert param in reached.as_sdk_kwargs() + document = cast(dict[str, object], reached.canonical_input()["document"]) + assert document == {"type": "document_url", "document_url": structured_pdf_data_uri()} + + +@pytest.mark.parametrize("target_name", ("mistral-ocr", "azure-mistral", "vertex-mistral")) +def test_mistral_recording_targets_reach_every_transport_branch(target_name: str) -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + target: Final = next(candidate for candidate in targets if candidate.name == target_name) + + for transport in ( + ("image_url", "remote"), + ("image_url", "data"), + ("document_url", "remote"), + ("document_url", "data"), + ): + reached = _find_input( + target.strategy, + lambda case_input, expected=transport: _document_transport(case_input) == expected, + ) + assert _document_transport(reached) == transport + + +def test_vertex_deepseek_recording_reaches_documented_image_branch() -> None: + targets: Final = discover_targets( + { + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + target: Final = next(candidate for candidate in targets if candidate.name == "vertex-deepseek") + case_input: Final = _find_input( + target.strategy, + lambda candidate: _document_transport(candidate) == ("image_url", "data"), + ) + + assert _document_transport(case_input) == ("image_url", "data") + + +def test_only_intentional_provider_failures_are_fixed_inputs() -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "REDUCTO_API_KEY": "reducto-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "AZURE_DOCUMENT_INTELLIGENCE_API_KEY": "document-secret", + "AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT": "https://document.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + + expected: Final[dict[str, tuple[OcrSdkInputBase, ...]]] = { + "mistral-ocr": MISTRAL_PROVIDER_REJECTED_INPUTS, + "azure-mistral": AZURE_MISTRAL_PROVIDER_REJECTED_INPUTS, + "azure-document-intelligence": AZURE_DOCUMENT_INTELLIGENCE_PROVIDER_REJECTED_INPUTS, + "vertex-mistral": vertex_mistral_provider_rejected_inputs("project-1", "us-central1", _INLINE_IMAGE_DATA_URI), + "vertex-deepseek": vertex_deepseek_provider_rejected_inputs("project-1", "us-central1", _INLINE_IMAGE_DATA_URI), + "reducto-v3": REDUCTO_V3_PROVIDER_REJECTED_INPUTS, + "reducto-legacy": REDUCTO_LEGACY_PROVIDER_REJECTED_INPUTS, + } + + assert {target.name for target in targets} == expected.keys() + for target in targets: + assert target.required_inputs == expected[target.name] + generated: Final = generate_case_inputs(target.strategy, examples=20) + assert all(case_input not in target.required_inputs for case_input in generated) + + +def test_mistral_adapters_preserve_omitted_optional_params() -> None: + targets: Final = discover_targets( + { + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + + baselines: Final = tuple( + _find_input( + target.strategy, + lambda case_input: _MISTRAL_PARAMS.isdisjoint(case_input.as_sdk_kwargs()), + ) + for target in targets + ) + assert all(_MISTRAL_PARAMS.isdisjoint(baseline.as_sdk_kwargs()) for baseline in baselines) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py new file mode 100644 index 00000000000..bedbdeb6a13 --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py @@ -0,0 +1,456 @@ +from __future__ import annotations + +import asyncio +import sys +import traceback +from collections.abc import Awaitable, Callable, Coroutine, Generator +from contextlib import contextmanager +from dataclasses import dataclass +from enum import Enum +from pathlib import Path +from typing import Final, cast + +import pytest + +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from litellm.rust_bridge import get_native_bridge +from litellm.rust_bridge import ocr as rust_ocr_bridge +from litellm.rust_bridge.ocr import RustAocr, RustOcr +from .....shared.parity.compare import assert_model_parity, assert_parity, assert_request_parity +from .....shared.parity.fixtures.store import recorded_fixtures +from .....shared.parity.inprocess import run_in_process +from .....shared.parity.models import ( + SDKCommand, + SDKError, + SDKReport, + SDKSuccess, + WorkerFailure, + WorkerResult, + WorkerSuccess, + sdk_error_report, +) +from .....shared.parity.replay import replay_server +from .....shared.parity.runner import ( + ExecutionVariant, + SubprocessRunner, + SubprocessWorker, + execution_worker_pair, + parity_worker_main, + run_execution, +) +from .fixtures.config import configured_fixture_directory +from .fixtures.models import OcrParityCase, OcrSdkInput + +API_KEY: Final = "test-key" +PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback" +PYTHON_VARIANT: Final = ExecutionVariant(name="Python", environment=(("LITELLM_USE_RUST_OCR", "0"),)) +RUST_VARIANT: Final = ExecutionVariant(name="Rust", environment=(("LITELLM_USE_RUST_OCR", "1"),)) + + +class SDKRoute(str, Enum): + OCR = "ocr" + AOCR = "aocr" + + +@dataclass(frozen=True, slots=True) +class InvalidOcrCase: + name: str + model: str + document: object + expected_exception_type: str + expected_status_code: int + expected_message: str + extra_kwargs: tuple[tuple[str, object], ...] = () + expected_rust_calls: int = 0 + + +INVALID_OCR_CASES: Final = ( + InvalidOcrCase( + name="unsupported_provider", + model="openai/gpt-4o", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="OCR is not supported for provider: openai", + ), + InvalidOcrCase( + name="unsupported_reducto_model", + model="reducto/parse-v4", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="OCR is not supported for provider: reducto", + ), + InvalidOcrCase( + name="unknown_provider_prefix", + model="not_a_provider/model", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, + expected_message="LLM Provider NOT provided", + ), + InvalidOcrCase( + name="non_object_document", + model="mistral/mistral-ocr-latest", + document=[], + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="document must be a dict", + ), + InvalidOcrCase( + name="missing_document_type", + model="mistral/mistral-ocr-latest", + document={}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid document type: None", + ), + InvalidOcrCase( + name="unsupported_document_type", + model="mistral/mistral-ocr-latest", + document={"type": "text", "text": "not a document"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid document type: text", + ), + InvalidOcrCase( + name="missing_document_url", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Document URL is required", + expected_rust_calls=1, + ), + InvalidOcrCase( + name="missing_image_url", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "image_url"}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Document URL is required", + expected_rust_calls=1, + ), + InvalidOcrCase( + name="invalid_request_format", + model="mistral/mistral-ocr-latest", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.UnsupportedParamsError", + expected_status_code=400, + expected_message="Invalid `req_format`: 'bogus'", + extra_kwargs=(("req_format", "bogus"),), + ), + InvalidOcrCase( + name="invalid_document_intelligence_pages", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="`pages` integers must be >= 0", + extra_kwargs=(("pages", [-1]),), + ), + InvalidOcrCase( + name="invalid_document_intelligence_features", + model="azure_ai/doc-intelligence/prebuilt-read", + document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + expected_exception_type="litellm.exceptions.APIConnectionError", + expected_status_code=500, + expected_message="Invalid `features` for Azure Document Intelligence", + extra_kwargs=(("features", [1]),), + ), +) + + +def _call_kwargs(sdk_input: OcrSdkInput, mock_url: str, route: SDKRoute) -> dict[str, object]: + return { + **sdk_input.as_sdk_kwargs(), + "api_base": mock_url, + "api_key": API_KEY, + "extra_headers": {"x-litellm-parity-route": route.value}, + } + + +def _execute_sdk_call( + call_kwargs: dict[str, object], + route: SDKRoute, + event_loop: asyncio.AbstractEventLoop, +) -> SDKReport: + import litellm + + try: + if route is SDKRoute.OCR: + sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr) + response: Final = sync_route(**call_kwargs) + return SDKSuccess(response=response.model_dump(mode="json")) + async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr) + async_response: Final = event_loop.run_until_complete(async_route(**call_kwargs)) + return SDKSuccess(response=async_response.model_dump(mode="json")) + except Exception as error: + return sdk_error_report(error) + + +def _execute_sdk_case( + sdk_input: OcrSdkInput, + route: SDKRoute, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> SDKReport: + call_kwargs: Final = _call_kwargs(sdk_input, mock_url, route) + return _execute_sdk_call(call_kwargs, route, event_loop) + + +def _execute_recorded_sdk_case( + sdk_input: OcrSdkInput, + route: SDKRoute, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> OCRResponse | SDKError: + import litellm + + call_kwargs: Final = _call_kwargs(sdk_input, mock_url, route) + try: + if route is SDKRoute.OCR: + sync_route: Final = cast(Callable[..., OCRResponse], litellm.ocr) + return sync_route(**call_kwargs) + async_route: Final = cast(Callable[..., Coroutine[object, object, OCRResponse]], litellm.aocr) + return event_loop.run_until_complete(async_route(**call_kwargs)) + except Exception as error: + return sdk_error_report(error) + + +def _execute_invalid_sdk_case( + case: InvalidOcrCase, + route: SDKRoute, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> SDKReport: + call_kwargs: Final = { + "model": case.model, + "document": case.document, + "api_base": mock_url, + "api_key": API_KEY, + "extra_headers": {"x-litellm-parity-route": route.value}, + **dict(case.extra_kwargs), + } + return _execute_sdk_call(call_kwargs, route, event_loop) + + +class _RustOcrSpy: + def __init__(self, delegate: RustOcr) -> None: + self.delegate: Final = delegate + self.calls = 0 + + def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: + self.calls += 1 + return self.delegate( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=timeout_seconds, + ) + + +class _RustAocrSpy: + def __init__(self, delegate: RustAocr) -> None: + self.delegate: Final = delegate + self.calls = 0 + + async def __call__( + self, + model: str, + document: dict[str, object], + api_key: str | None, + api_base: str | None, + custom_llm_provider: str | None, + extra_headers: dict[str, object] | None, + optional_params: dict[str, object], + timeout_seconds: float | None, + ) -> dict[str, object]: + self.calls += 1 + result: Final[Awaitable[dict[str, object]]] = self.delegate( + model=model, + document=document, + api_key=api_key, + api_base=api_base, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + optional_params=optional_params, + timeout_seconds=timeout_seconds, + ) + return await result + + +@contextmanager +def _restore_rust_ocr_state() -> Generator[None]: + enabled: Final = rust_ocr_bridge.rust_ocr_enabled() + ocr_impl: Final = rust_ocr_bridge._rust_ocr_impl # pyright: ignore[reportPrivateUsage] # preserve injected test binding + aocr_impl: Final = rust_ocr_bridge._rust_aocr_impl # pyright: ignore[reportPrivateUsage] # preserve injected test binding + try: + yield + finally: + rust_ocr_bridge.use_litellm_rust(enabled, ocr=ocr_impl, aocr=aocr_impl) + + +def _native_spies() -> tuple[_RustOcrSpy, _RustAocrSpy]: + native_bridge: Final = get_native_bridge() + if native_bridge is None: + pytest.fail("native Rust bridge is required for OCR parity testing") + sync_spy: Final = _RustOcrSpy(cast(RustOcr, getattr(native_bridge, "ocr"))) + async_spy: Final = _RustAocrSpy(cast(RustAocr, getattr(native_bridge, "aocr"))) + return sync_spy, async_spy + + +@pytest.fixture(scope="module") +def sdk_workers() -> Generator[tuple[SubprocessWorker, SubprocessWorker]]: + runner: Final = SubprocessRunner( + entrypoint=Path(__file__), + baseline_user_agent=PYTHON_HTTP_SENTINEL, + route_label="OCR", + ) + with execution_worker_pair(runner, PYTHON_VARIANT, RUST_VARIANT) as workers: + yield workers + + +@pytest.fixture(scope="module") +def startup_ocr_fixture() -> OcrParityCase: + directory: Final = configured_fixture_directory() + fixtures: Final = recorded_fixtures(directory, OcrParityCase) + if not fixtures: + pytest.skip(f"no recorded fixtures in {directory}") + return fixtures[0] + + +@pytest.mark.parametrize("route", tuple(SDKRoute), ids=tuple(route.value for route in SDKRoute)) +def test_recorded_ocr_sdk_parity( + ocr_fixture: OcrParityCase, + route: SDKRoute, +) -> None: + sync_spy, async_spy = _native_spies() + event_loop: Final = asyncio.new_event_loop() + try: + with _restore_rust_ocr_state(), replay_server() as provider: + rust_ocr_bridge.use_litellm_rust(False, ocr=sync_spy, aocr=async_spy) + rust_ocr_bridge.use_litellm_rust(False) + python: Final = run_in_process( + provider, + ocr_fixture.provider_responses, + lambda mock_url: _execute_recorded_sdk_case(ocr_fixture.litellm_input, route, mock_url, event_loop), + ) + assert sync_spy.calls == 0 + assert async_spy.calls == 0 + + rust_ocr_bridge.use_litellm_rust(True) + rust: Final = run_in_process( + provider, + ocr_fixture.provider_responses, + lambda mock_url: _execute_recorded_sdk_case(ocr_fixture.litellm_input, route, mock_url, event_loop), + ) + finally: + event_loop.close() + + assert sync_spy.calls == (1 if route is SDKRoute.OCR else 0) + assert async_spy.calls == (1 if route is SDKRoute.AOCR else 0) + assert_request_parity(python.requests, rust.requests) + if any(response.status_code >= 400 for response in ocr_fixture.provider_responses): + assert isinstance(python.response, SDKError) + if isinstance(python.response, SDKError): + assert python.response == rust.response + else: + assert isinstance(rust.response, OCRResponse) + assert_model_parity(python.response, rust.response) + + +@pytest.mark.parametrize("case", INVALID_OCR_CASES, ids=tuple(case.name for case in INVALID_OCR_CASES)) +@pytest.mark.parametrize("route", tuple(SDKRoute), ids=tuple(route.value for route in SDKRoute)) +def test_invalid_ocr_sdk_parity(case: InvalidOcrCase, route: SDKRoute) -> None: + sync_spy, async_spy = _native_spies() + event_loop: Final = asyncio.new_event_loop() + try: + with _restore_rust_ocr_state(), replay_server() as provider: + rust_ocr_bridge.use_litellm_rust(False, ocr=sync_spy, aocr=async_spy) + rust_ocr_bridge.use_litellm_rust(False) + python: Final = run_in_process( + provider, + (), + lambda mock_url: _execute_invalid_sdk_case(case, route, mock_url, event_loop), + ) + assert sync_spy.calls == 0 + assert async_spy.calls == 0 + + rust_ocr_bridge.use_litellm_rust(True) + rust: Final = run_in_process( + provider, + (), + lambda mock_url: _execute_invalid_sdk_case(case, route, mock_url, event_loop), + ) + finally: + event_loop.close() + + assert sync_spy.calls == (case.expected_rust_calls if route is SDKRoute.OCR else 0) + assert async_spy.calls == (case.expected_rust_calls if route is SDKRoute.AOCR else 0) + assert python.requests == () + assert rust.requests == () + assert python.response == rust.response + assert isinstance(python.response, SDKError) + assert python.response.exception_type == case.expected_exception_type + assert python.response.status_code == case.expected_status_code + assert case.expected_message in python.response.message + + +def test_ocr_subprocess_startup_smoke( + startup_ocr_fixture: OcrParityCase, + tmp_path: Path, + sdk_workers: tuple[SubprocessWorker, SubprocessWorker], +) -> None: + case_file: Final = tmp_path / "ocr-startup-smoke.json" + case_file.write_text(startup_ocr_fixture.model_dump_json(indent=2, exclude_unset=True), encoding="utf-8") + python_worker, rust_worker = sdk_workers + python: Final = run_execution( + python_worker, + case_file, + SDKRoute.OCR.value, + startup_ocr_fixture.provider_responses, + ) + rust: Final = run_execution( + rust_worker, + case_file, + SDKRoute.OCR.value, + startup_ocr_fixture.provider_responses, + ) + + assert_parity(python, rust, PYTHON_HTTP_SENTINEL) + + +def _execute_worker_command( + command_json: str, + mock_url: str, + event_loop: asyncio.AbstractEventLoop, +) -> WorkerResult: + try: + command: Final = SDKCommand.model_validate_json(command_json) + case_file: Final = Path(command.case_file) + route: Final = SDKRoute(command.route) + case: Final = OcrParityCase.model_validate_json(case_file.read_text(encoding="utf-8")) + return WorkerSuccess(report=_execute_sdk_case(case.litellm_input, route, mock_url, event_loop)) + except Exception: + return WorkerFailure(error=traceback.format_exc()) + + +if __name__ == "__main__": + if len(sys.argv) != 3 or sys.argv[1] != "--parity-worker": + raise SystemExit("usage: test_sdk_parity.py --parity-worker MOCK_URL") + parity_worker_main(_execute_worker_command, sys.argv[2]) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/__init__.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/e2e_parity/strategy.json b/tests/rust-python-harness/strategies/e2e_parity/strategy.json new file mode 100644 index 00000000000..d791b9373aa --- /dev/null +++ b/tests/rust-python-harness/strategies/e2e_parity/strategy.json @@ -0,0 +1,50 @@ +{ + "order": 10, + "id": "e2e_parity", + "label": "End-to-end parity", + "description": "Compare observable Python and Rust SDK behavior over generated and recorded inputs.", + "functions": { + "ocr": { + "coverage": "partial", + "selectors": [ + "tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py", + "tests/test_litellm/ocr/test_rust_bridge.py" + ], + "note": "Recorded sync/async SDK parity; invalid-model provider errors differ, and Reducto lacks a Rust contract." + }, + "messages": { + "coverage": "partial", + "selectors": [ + "tests/test_litellm/anthropic_interface/test_rust_bridge_messages.py" + ], + "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added." + }, + "responses": { + "coverage": "partial", + "selectors": [ + "tests/test_litellm/responses/test_rust_bridge_websocket.py" + ], + "note": "Covers the websocket bridge; full responses parity is still being added." + }, + "count_tokens": { + "coverage": "planned", + "selectors": [], + "note": "No Rust count_tokens parity test is present yet." + }, + "chat_completions": { + "coverage": "partial", + "selectors": [ + "tests/test_litellm/rust_bridge/test_chat_completions.py" + ], + "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added." + }, + "transcription": { + "coverage": "partial", + "selectors": [ + "tests/test_litellm/test_audio_transcription_rust_bridge.py" + ], + "note": "Bridge coverage exists; frozen-oracle fuzz parity is still being added." + } + }, + "gateway": {} +} diff --git a/tests/rust-python-harness/existing_e2e_test_sdk/README.md b/tests/rust-python-harness/strategies/existing_e2e_test_sdk/README.md similarity index 100% rename from tests/rust-python-harness/existing_e2e_test_sdk/README.md rename to tests/rust-python-harness/strategies/existing_e2e_test_sdk/README.md diff --git a/tests/rust-python-harness/strategies/existing_e2e_test_sdk/__init__.py b/tests/rust-python-harness/strategies/existing_e2e_test_sdk/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/existing_e2e_test_sdk/runner.py b/tests/rust-python-harness/strategies/existing_e2e_test_sdk/runner.py new file mode 100644 index 00000000000..f5ea17735fc --- /dev/null +++ b/tests/rust-python-harness/strategies/existing_e2e_test_sdk/runner.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from ...shared.reporting.models import HarnessCase, HarnessRun +from ...shared.reporting.pytest_runner import UpdateCallback, run_pytest + + +def run( + cases: Sequence[HarnessCase], + repo_root: Path, + on_update: UpdateCallback, + pytest_args: Sequence[str] = (), +) -> tuple[int, HarnessRun]: + return run_pytest(cases, repo_root, on_update, pytest_args) + + +def main(argv: Sequence[str] | None = None) -> int: + from ...cli import main as harness_main + + return harness_main(argv, strategy_id="existing_e2e_test_sdk") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/rust-python-harness/existing_e2e_test_sdk/strategy.json b/tests/rust-python-harness/strategies/existing_e2e_test_sdk/strategy.json similarity index 100% rename from tests/rust-python-harness/existing_e2e_test_sdk/strategy.json rename to tests/rust-python-harness/strategies/existing_e2e_test_sdk/strategy.json diff --git a/tests/rust-python-harness/strategies/trace_parity/README.md b/tests/rust-python-harness/strategies/trace_parity/README.md new file mode 100644 index 00000000000..6520a510112 --- /dev/null +++ b/tests/rust-python-harness/strategies/trace_parity/README.md @@ -0,0 +1,5 @@ +# Trace Parity + +Run independently with `uv run python -m tests.rust-python-harness.strategies.trace_parity.runner --plain`. Configure SDK and gateway selectors in `strategy.json`; keep API-specific execution and fixtures in their owning surface folder + +See [the harness guide](../../README.md) for coverage status and shared comparison tools diff --git a/tests/rust-python-harness/strategies/trace_parity/__init__.py b/tests/rust-python-harness/strategies/trace_parity/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py b/tests/rust-python-harness/strategies/trace_parity/gateway/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/trace_parity/runner.py b/tests/rust-python-harness/strategies/trace_parity/runner.py new file mode 100644 index 00000000000..127bf5dce40 --- /dev/null +++ b/tests/rust-python-harness/strategies/trace_parity/runner.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from collections.abc import Sequence +from pathlib import Path + +from ...shared.reporting.models import HarnessCase, HarnessRun +from ...shared.reporting.pytest_runner import UpdateCallback, run_pytest + + +def run( + cases: Sequence[HarnessCase], + repo_root: Path, + on_update: UpdateCallback, + pytest_args: Sequence[str] = (), +) -> tuple[int, HarnessRun]: + return run_pytest(cases, repo_root, on_update, pytest_args) + + +def main(argv: Sequence[str] | None = None) -> int: + from ...cli import main as harness_main + + return harness_main(argv, strategy_id="trace_parity") + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/rust-python-harness/strategies/trace_parity/sdk/__init__.py b/tests/rust-python-harness/strategies/trace_parity/sdk/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/rust-python-harness/strategies/trace_parity/strategy.json b/tests/rust-python-harness/strategies/trace_parity/strategy.json new file mode 100644 index 00000000000..9b67d8570cc --- /dev/null +++ b/tests/rust-python-harness/strategies/trace_parity/strategy.json @@ -0,0 +1,33 @@ +{ + "order": 20, + "id": "trace_parity", + "label": "Trace parity", + "description": "Compare mapped operations, call counts, and required execution ordering.", + "functions": { + "ocr": { + "coverage": "planned", + "selectors": [] + }, + "messages": { + "coverage": "planned", + "selectors": [] + }, + "chat_completions": { + "coverage": "planned", + "selectors": [] + }, + "responses": { + "coverage": "planned", + "selectors": [] + }, + "count_tokens": { + "coverage": "planned", + "selectors": [] + }, + "transcription": { + "coverage": "planned", + "selectors": [] + } + }, + "gateway": {} +} diff --git a/tests/rust-python-harness/strategies/unit_tests/README.md b/tests/rust-python-harness/strategies/unit_tests/README.md new file mode 100644 index 00000000000..bd37072ae4b --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/README.md @@ -0,0 +1,7 @@ +# Unit tests + +Run independently with `uv run python -m tests.rust-python-harness.strategies.unit_tests.runner --plain`. Configure a `unit_suite` for each mapped API in `strategy.json` + +The runner combines mapping validation, Python tests in separate verified backend processes, and Cargo tests. It reports missing and ambiguous counterparts. Native Rust tests and existing Python tests stay in their original locations + +See [the suite format](../../README.md#configure-cases) for configuration. No complete API mapping is configured yet diff --git a/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json index e617ddf8f94..1ceb79b52bc 100644 --- a/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json +++ b/tests/rust-python-harness/strategies/unit_tests/ledgers/ocr/ocr_test_ledger.json @@ -20,1065 +20,192 @@ "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs" ], "entries": [ - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_should_encode_azure_document_intelligence_model_id", - "status": "unmapped", - "reason": "model-id URL percent-encoding has no Rust test; Rust only tests pages/features query building" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_should_reject_dot_segment_azure_document_intelligence_model_id", - "status": "unmapped", - "reason": "model-id dot-segment validation has no Rust test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_transform_ocr_response_preserves_azure_native_fields", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", - "rust_test": "document_intelligence_response_normalizes_pages", - "justification": "both assert page markdown, dimension (inch-to-pixel) normalization, and usage_info.pages_processed from the same Azure succeeded response shape" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_async_transform_ocr_response_preserves_azure_native_fields", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", - "rust_test": "document_intelligence_response_normalizes_pages", - "justification": "async twin of the sync case above, same underlying transform is exercised on the Rust side" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_transform_ocr_response_tolerates_missing_native_fields", - "status": "unmapped", - "reason": "tables/keyValuePairs absence tolerance is not asserted by the Rust response test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_transform_ocr_response_non_succeeded_status_raises", - "status": "unmapped", - "reason": "no Rust test asserts on a non-succeeded Azure DI status" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_get_supported_ocr_params_includes_features", - "status": "unmapped", - "reason": "supported-params list content has no Rust equivalent for Azure" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_transform_ocr_response_native_format_carries_raw_operation", - "status": "unmapped", - "reason": "native req_format raw-operation passthrough is not tested in Rust" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_async_transform_ocr_response_native_format_carries_raw_operation", - "status": "unmapped", - "reason": "native req_format raw-operation passthrough is not tested in Rust" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_transform_ocr_response_default_format_omits_raw_operation", - "status": "unmapped", - "reason": "req_format gating of raw-operation output has no Rust test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_map_ocr_params_passes_through_req_format", - "status": "unmapped", - "reason": "req_format passthrough in map_ocr_params has no Rust test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_map_ocr_params_rejects_unknown_req_format_as_bad_request", - "status": "unmapped", - "reason": "req_format validation error path has no Rust test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_get_complete_url_omits_req_format_query_param", - "status": "unmapped", - "reason": "no Rust test asserts req_format is excluded from the built URL" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_map_ocr_params_features", - "status": "unmapped", - "reason": "features-string normalization in map_ocr_params has no Rust test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_map_ocr_params_empty_features_list_omitted", - "status": "unmapped", - "reason": "empty-features omission has no Rust test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_map_ocr_params_invalid_features_raises", - "status": "unmapped", - "reason": "features validation error path has no Rust test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_get_complete_url_appends_features_query", - "status": "unmapped", - "reason": "features query-param construction has no Rust test" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_get_complete_url_combines_pages_and_features", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", - "rust_test": "document_intelligence_url_normalizes_zero_based_pages", - "justification": "both assert 0-based, duplicate page indices are deduped, sorted, and rewritten 1-based into the request URL" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_validate_environment_uses_subscription_key", - "status": "unmapped", - "reason": "Python-side header derivation from litellm_params; Rust's poll test only checks the header is present, not how it was resolved" - }, - { - "python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", - "python_test": "test_validate_environment_falls_back_to_entra_token", - "status": "unmapped", - "reason": "Entra bearer-token fallback logic has no Rust test" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", - "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_doc_intelligence_route", - "status": "unmapped", - "reason": "model-route string matching is Python-only dispatch logic" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", - "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_documentintelligence_and_is_case_insensitive", - "status": "unmapped", - "reason": "model-route string matching is Python-only dispatch logic" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", - "python_test": "TestIsAzureDocumentIntelligenceModel::test_does_not_match_mistral_route", - "status": "unmapped", - "reason": "model-route string matching is Python-only dispatch logic" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", - "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", - "status": "unmapped", - "reason": "api_base resolution from the secret manager runs before the Rust bridge is called, no Rust test exists for it" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", - "python_test": "TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", - "status": "unmapped", - "reason": "api_base precedence resolution is Python-only" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", - "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", - "status": "unmapped", - "reason": "api_base precedence resolution is Python-only" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_use_litellm_rust_toggles_flag", - "status": "unmapped", - "reason": "bridge-plumbing: Python-side feature-flag toggle, no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_env_var_enables_rust_ocr", - "status": "unmapped", - "reason": "bridge-plumbing: Python-side env-var flag gating" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_load_rust_ocr_returns_injected_impl", - "status": "unmapped", - "reason": "bridge-plumbing: dependency-injection test hook, not provider behavior" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_native_bridge_loader_returns_none_when_extension_absent", - "status": "unmapped", - "reason": "bridge-plumbing: native-extension import/loader fallback" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_native_bridge_loader_caches_absent_extension", - "status": "unmapped", - "reason": "bridge-plumbing: loader caching behavior" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_native_bridge_available_reflects_loader", - "status": "unmapped", - "reason": "bridge-plumbing: loader availability check" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_load_rust_aocr_returns_injected_impl", - "status": "unmapped", - "reason": "bridge-plumbing: dependency-injection test hook" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_toggle_without_ocr_arg_preserves_injected_impl", - "status": "unmapped", - "reason": "bridge-plumbing: injected-impl state retention regression" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_explicit_ocr_none_clears_injected_impl", - "status": "unmapped", - "reason": "bridge-plumbing: injected-impl clearing behavior" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_load_rust_ocr_none_when_extension_absent", - "status": "unmapped", - "reason": "bridge-plumbing: degrade path when the native extension is missing" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_load_rust_ocr_uses_compiled_extension", - "status": "unmapped", - "reason": "bridge-plumbing: native module resolution" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_timeout_to_seconds_handles_float_timeout_and_none", - "status": "unmapped", - "reason": "bridge-plumbing: Python-side timeout normalization helper" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_bridge_wrapper_forwards_prepared_args_and_wraps_response", - "status": "unmapped", - "reason": "bridge-plumbing: wrapper argument forwarding, asserted against a fake bridge not the real Rust code" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response", - "status": "unmapped", - "reason": "bridge-plumbing: async wrapper argument forwarding" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_run_rust_ocr_prepares_request_and_wraps_response", - "status": "unmapped", - "reason": "bridge-plumbing: request preparation and response wrapping in Python" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_run_rust_ocr_resolves_key_via_secret_manager_when_missing", - "status": "unmapped", - "reason": "secret-manager: API key resolution happens in Python before the bridge is invoked" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_run_rust_ocr_prefers_explicit_key_over_resolver", - "status": "unmapped", - "reason": "secret-manager: key precedence resolution" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_run_rust_ocr_uses_provider_api_key_env_var", - "status": "unmapped", - "reason": "secret-manager: provider-specific env var name resolution" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_prepare_rust_ocr_call_forwards_vertex_routing_metadata", - "status": "unmapped", - "reason": "secret-manager: vertex routing metadata merge happens in Python" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager", - "status": "unmapped", - "reason": "secret-manager: vertex project/location resolution" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager", - "status": "unmapped", - "reason": "secret-manager: azure_ai api_base resolution" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint", - "status": "unmapped", - "reason": "secret-manager: doc-intelligence endpoint resolution" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_run_rust_ocr_runs_pre_call_logging", - "status": "unmapped", - "reason": "bridge-plumbing: Python logging-object pre_call invocation" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_routes_to_rust_when_enabled", - "status": "unmapped", - "reason": "bridge-plumbing: routing to a fake bridge, not the real Rust transform" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_routes_azure_ai_to_rust_when_enabled", - "status": "unmapped", - "reason": "bridge-plumbing: provider-prefix stripping before routing" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_rust_path_converts_file_document_before_bridge", - "status": "unmapped", - "reason": "file-normalization: raw-bytes-to-data-URI conversion happens in Python before the bridge call" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_exception_type_uses_resolved_provider_context", - "status": "unmapped", - "reason": "bridge-plumbing: Python exception-type mapping on bridge failure" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_aocr_routes_to_async_rust_when_enabled", - "status": "unmapped", - "reason": "bridge-plumbing: async routing to a fake bridge" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_aocr_exception_type_uses_resolved_provider_context", - "status": "unmapped", - "reason": "bridge-plumbing: async exception-type mapping on bridge failure" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_does_not_route_to_rust_when_disabled", - "status": "unmapped", - "reason": "bridge-plumbing: Python control flow for the toggle-disabled branch, no Rust-owned behavior runs" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_falls_back_to_python_when_bridge_unavailable", - "status": "unmapped", - "reason": "bridge-plumbing: Python-only fallback when the compiled Rust extension is absent, Rust cannot test its own absence" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_forwards_timeout_to_rust", - "status": "unmapped", - "reason": "bridge-plumbing: asserts the Python call site forwards a timeout kwarg, Rust receives an already-constructed request" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_passes_default_request_timeout_to_rust", - "status": "unmapped", - "reason": "bridge-plumbing: asserts the Python call site supplies a default timeout kwarg, no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_ocr_provider_configs_expose_api_key_env_vars", - "status": "unmapped", - "reason": "asserts per-provider get_api_key_env_var() strings; the closest Rust test (ocr_dispatch_supports_migrated_providers) asserts provider dispatch/param resolution instead, not API key env var names" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_pdf_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection is Python-only preprocessing before the bridge call" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_png_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_jpg_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_jpeg_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_gif_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_webp_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_tiff_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_tif_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_detect_bmp_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_be_case_insensitive", - "status": "unmapped", - "reason": "file-normalization: MIME detection case handling" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestGetMimeType::test_should_fallback_for_unknown_extension", - "status": "unmapped", - "reason": "file-normalization: MIME detection fallback" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pdf_pathlib_path_to_document_url", - "status": "unmapped", - "reason": "file-normalization: local-path-to-data-URI conversion happens in Python" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_image_pathlib_path_to_image_url", - "status": "unmapped", - "reason": "file-normalization: local-path-to-data-URI conversion" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_reject_bare_str_path", - "status": "unmapped", - "reason": "file-normalization: arbitrary-file-read guard on bare str paths" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pathlib_path", - "status": "unmapped", - "reason": "file-normalization: local-path-to-data-URI conversion" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes", - "status": "unmapped", - "reason": "file-normalization: raw-bytes-to-data-URI conversion" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_explicit_mime_type", - "status": "unmapped", - "reason": "file-normalization: explicit MIME override on raw bytes" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_image_mime_type", - "status": "unmapped", - "reason": "file-normalization: explicit MIME override on raw bytes" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object", - "status": "unmapped", - "reason": "file-normalization: file-like-object conversion" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object_with_name", - "status": "unmapped", - "reason": "file-normalization: file-like-object name-based MIME detection" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_missing_file_field", - "status": "unmapped", - "reason": "file-normalization: missing-field validation" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_nonexistent_pathlib_path", - "status": "unmapped", - "reason": "file-normalization: missing-file validation" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_empty_file", - "status": "unmapped", - "reason": "file-normalization: empty-file validation" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_unsupported_type", - "status": "unmapped", - "reason": "file-normalization: unsupported input type validation" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_invalid_mime_type", - "status": "unmapped", - "reason": "file-normalization: MIME-type injection validation" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestConvertFileDocumentToUrlDocument::test_should_override_mime_type_for_pathlib_path", - "status": "unmapped", - "reason": "file-normalization: explicit MIME override precedence" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_build_document_url_for_pdf", - "status": "unmapped", - "reason": "file-normalization: multipart upload conversion" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_png", - "status": "unmapped", - "reason": "file-normalization: multipart upload conversion" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_jpeg", - "status": "unmapped", - "reason": "file-normalization: multipart upload conversion" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_octet_stream", - "status": "unmapped", - "reason": "file-normalization: filename-based MIME fallback" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_none", - "status": "unmapped", - "reason": "file-normalization: filename-based MIME fallback" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_fallback_to_octet_stream_for_unknown", - "status": "unmapped", - "reason": "file-normalization: default MIME fallback" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_preserve_base64_content_correctly", - "status": "unmapped", - "reason": "file-normalization: binary round-trip through base64" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_from_content_type", - "status": "unmapped", - "reason": "file-normalization: content-type parameter stripping" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_with_multiple_params", - "status": "unmapped", - "reason": "file-normalization: content-type parameter stripping" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestProxySecurityGuard::test_should_reject_file_type_document_in_json_body", - "status": "unmapped", - "reason": "proxy-layer JSON-body file-type guard, a different mechanism than Rust's URL-fetch SSRF guard" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestProxySecurityGuard::test_should_accept_document_url_type_in_json_body", - "status": "unmapped", - "reason": "proxy-layer JSON-body parsing" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestProxySecurityGuard::test_should_raise_on_invalid_json_body", - "status": "unmapped", - "reason": "proxy-layer JSON-body parsing error path" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", - "python_test": "TestProxySecurityGuard::test_should_ignore_document_form_field_injection", - "status": "unmapped", - "reason": "proxy-layer multipart form-field injection guard, a different mechanism than Rust's URL-fetch SSRF guard" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestGetSupportedOcrParams::test_extract_header_in_supported_params", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "extract_header_is_a_supported_ocr_param", - "justification": "both assert extract_header appears in the Mistral supported OCR params list" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestGetSupportedOcrParams::test_extract_footer_in_supported_params", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "extract_footer_is_a_supported_ocr_param", - "justification": "both assert extract_footer appears in the Mistral supported OCR params list" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestGetSupportedOcrParams::test_existing_params_still_present", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "existing_ocr_params_remain_supported", - "justification": "both assert the previously supported params are still present in the supported list" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestMapOcrParams::test_extract_header_passed_through", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "map_ocr_params_forwards_extract_header", - "justification": "both assert extract_header alone survives map_ocr_params unchanged" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestMapOcrParams::test_extract_footer_passed_through", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "map_ocr_params_forwards_extract_footer", - "justification": "both assert extract_footer alone survives map_ocr_params unchanged" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestMapOcrParams::test_extract_header_and_footer_together", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "map_ocr_params_forwards_extract_header_and_footer", - "justification": "both assert header and footer passed together are both forwarded with their given values" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestMapOcrParams::test_unknown_param_is_dropped", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "map_ocr_params_drops_unknown_params", - "justification": "both assert an unrecognized param key is dropped while a known one is kept" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestNewSupportedParams::test_new_param_in_supported_list", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "new_ocr_params_are_supported", - "justification": "both assert each OCR4 param is in the supported list" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestNewParamsMapOcr::test_new_param_passed_through", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "map_ocr_params_forwards_new_ocr_params", - "justification": "both assert each OCR4 param/value pair survives map_ocr_params unchanged" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestTransformOcrRequest::test_param_included_in_request_body", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "transform_ocr_request_includes_each_optional_param", - "justification": "both assert each optional param value lands in the built request body alongside model/document with no files" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestTransformOcrRequest::test_multiple_new_params_together", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "transform_ocr_request_includes_multiple_new_params", - "justification": "both assert multiple OCR4 params passed together all land in the same request body" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "transform_ocr_response_preserves_blocks_and_confidence_scores", - "justification": "both assert blocks and confidence_scores survive the OCR response transform on the returned page" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", - "python_test": "TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", - "status": "mapped", - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "transform_ocr_response_preserves_ocr4_page_fields", - "justification": "both assert tables, hyperlinks, header and footer survive the OCR response transform on the returned page" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_model_info_ocr4_price", - "status": "unmapped", - "reason": "cost-calc: pricing/model-info lookup is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_ocr4_cost_scales_with_pages", - "status": "unmapped", - "reason": "cost-calc: per-page pricing math is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_ocr3_pricing_entry", - "status": "unmapped", - "reason": "cost-calc: cost-map JSON entry validation is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_ocr3_model_info_price", - "status": "unmapped", - "reason": "cost-calc: pricing/model-info lookup is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_ocr3_cost_scales_with_pages", - "status": "unmapped", - "reason": "cost-calc: per-page pricing math is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_ocr3_bills_ocr_and_annotation_pages_at_their_own_rates", - "status": "unmapped", - "reason": "cost-calc: mixed-rate billing math is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_ocr3_bills_annotation_only_response", - "status": "unmapped", - "reason": "cost-calc: annotation-only billing math is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_ocr3_bills_annotation_pages_when_pages_processed_missing", - "status": "unmapped", - "reason": "cost-calc: fallback billing math is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", - "python_test": "test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate", - "status": "unmapped", - "reason": "cost-calc: fallback billing math is Python-only" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", - "python_test": "test_rust_ocr_serves_default_format", - "status": "unmapped", - "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", - "python_test": "test_rust_ocr_skipped_for_native_format", - "status": "unmapped", - "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", - "python_test": "test_native_format_rejected_for_provider_without_support_as_bad_request", - "status": "unmapped", - "reason": "provider-support validation for req_format happens in Python" - }, - { - "python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", - "python_test": "test_unknown_format_rejected_for_provider_without_support_as_bad_request", - "status": "unmapped", - "reason": "req_format validation error path is Python-only" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestHandlerDiscovery::test_handler_discovered_for_ocr", - "status": "unmapped", - "reason": "guardrail-translation handler discovery is a Python proxy-layer concern" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestHandlerDiscovery::test_handler_discovered_for_aocr", - "status": "unmapped", - "reason": "guardrail-translation handler discovery is a Python proxy-layer concern" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestInputProcessing::test_process_document_url", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestInputProcessing::test_process_image_url", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestInputProcessing::test_process_no_document", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestInputProcessing::test_process_invalid_document", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestInputProcessing::test_input_blocking_guardrail", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestOutputProcessing::test_process_single_page", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestOutputProcessing::test_process_multiple_pages", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestOutputProcessing::test_process_empty_pages", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestOutputProcessing::test_process_page_with_empty_markdown", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestOutputProcessing::test_process_preserves_page_metadata", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestOutputProcessing::test_output_blocking_guardrail", - "status": "unmapped", - "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle" - }, - { - "python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", - "python_test": "TestPIIMaskingScenario::test_pii_masking_in_ocr_pages", - "status": "unmapped", - "reason": "PII redaction in the translation handler has no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", - "python_test": "test_should_read_req_format_from_header", - "status": "unmapped", - "reason": "proxy-layer header parsing has no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", - "python_test": "test_should_prefer_body_req_format_over_header", - "status": "unmapped", - "reason": "proxy-layer body-vs-header precedence has no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", - "python_test": "test_should_omit_req_format_when_header_absent", - "status": "unmapped", - "reason": "proxy-layer parsing has no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", - "python_test": "test_should_reject_unknown_req_format", - "status": "unmapped", - "reason": "proxy-layer validation has no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", - "python_test": "test_should_return_native_payload_with_litellm_response_headers", - "status": "unmapped", - "reason": "proxy-layer response construction has no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", - "python_test": "test_should_return_normalized_response_when_no_native_payload", - "status": "unmapped", - "reason": "proxy-layer response construction has no Rust equivalent" - }, - { - "python_file": "tests/test_litellm/ocr/test_rust_bridge.py", - "python_test": "test_explicit_false_overrides_process_enable", - "status": "unmapped", - "reason": "bridge-plumbing: asserts the explicit-per-request rust:False override wins over the Python process flag, no Rust-owned behavior runs" - } + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_encode_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id URL percent-encoding has no Rust test; Rust only tests pages/features query building"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_reject_dot_segment_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id dot-segment validation has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "both assert page markdown, dimension (inch-to-pixel) normalization, and usage_info.pages_processed from the same Azure succeeded response shape"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "async twin of the sync case above, same underlying transform is exercised on the Rust side"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_tolerates_missing_native_fields", "status": "unmapped", "reason": "tables/keyValuePairs absence tolerance is not asserted by the Rust response test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_non_succeeded_status_raises", "status": "unmapped", "reason": "no Rust test asserts on a non-succeeded Azure DI status"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_supported_ocr_params_includes_features", "status": "unmapped", "reason": "supported-params list content has no Rust equivalent for Azure"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_default_format_omits_raw_operation", "status": "unmapped", "reason": "req_format gating of raw-operation output has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_passes_through_req_format", "status": "unmapped", "reason": "req_format passthrough in map_ocr_params has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_rejects_unknown_req_format_as_bad_request", "status": "unmapped", "reason": "req_format validation error path has no Rust test"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_omits_req_format_query_param", "status": "unmapped", "reason": "no Rust test asserts req_format is excluded from the built URL"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_features", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_normalizes_features", "justification": "both normalize comma-separated feature names and whitespace; Python does this during parameter mapping and Rust during URL construction"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_empty_features_list_omitted", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_omits_empty_feature_list", "justification": "both omit empty feature lists from the outgoing request; Python removes the parameter and Rust omits the query field"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_invalid_features_raises", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_rejects_invalid_features", "justification": "both reject malformed feature values, including query injection, empty strings, and objects before sending the request"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_appends_features_query", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_normalizes_features", "justification": "both assert the selected feature names appear in the outgoing features query parameter"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_combines_pages_and_features", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_combines_pages_and_feature_list", "justification": "both combine zero-based pages [0, 1, 2] with keyValuePairs and languages into pages=1,2,3 and features=keyValuePairs,languages"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_uses_subscription_key", "status": "unmapped", "reason": "Python-side header derivation from litellm_params; Rust's poll test only checks the header is present, not how it was resolved"}, + {"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_falls_back_to_entra_token", "status": "unmapped", "reason": "Entra bearer-token fallback logic has no Rust test"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_doc_intelligence_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_documentintelligence_and_is_case_insensitive", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_does_not_match_mistral_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", "status": "unmapped", "reason": "api_base resolution from the secret manager runs before the Rust bridge is called, no Rust test exists for it"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"}, + + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_use_litellm_rust_toggles_flag", "status": "unmapped", "reason": "bridge-plumbing: Python-side feature-flag toggle, no Rust equivalent"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_env_var_enables_rust_ocr", "status": "unmapped", "reason": "bridge-plumbing: Python-side env-var flag gating"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_explicit_false_overrides_process_enable", "status": "unmapped", "reason": "Python request-level Rust opt-out overrides the process flag before any Rust implementation runs"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook, not provider behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_returns_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: native-extension import/loader fallback"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_caches_absent_extension", "status": "unmapped", "reason": "bridge-plumbing: loader caching behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_available_reflects_loader", "status": "unmapped", "reason": "bridge-plumbing: loader availability check"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_aocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_toggle_without_ocr_arg_preserves_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl state retention regression"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_explicit_ocr_none_clears_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl clearing behavior"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: degrade path when the native extension is missing"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_uses_compiled_extension", "status": "unmapped", "reason": "bridge-plumbing: native module resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_timeout_to_seconds_handles_float_timeout_and_none", "status": "unmapped", "reason": "bridge-plumbing: Python-side timeout normalization helper"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: wrapper argument forwarding, asserted against a fake bridge not the real Rust code"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: async wrapper argument forwarding"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prepares_request_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: request preparation and response wrapping in Python"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_resolves_key_via_secret_manager_when_missing", "status": "unmapped", "reason": "secret-manager: API key resolution happens in Python before the bridge is invoked"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prefers_explicit_key_over_resolver", "status": "unmapped", "reason": "secret-manager: key precedence resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_uses_provider_api_key_env_var", "status": "unmapped", "reason": "secret-manager: provider-specific env var name resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_forwards_vertex_routing_metadata", "status": "unmapped", "reason": "secret-manager: vertex routing metadata merge happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager", "status": "unmapped", "reason": "secret-manager: vertex project/location resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager", "status": "unmapped", "reason": "secret-manager: azure_ai api_base resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint", "status": "unmapped", "reason": "secret-manager: doc-intelligence endpoint resolution"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_runs_pre_call_logging", "status": "unmapped", "reason": "bridge-plumbing: Python logging-object pre_call invocation"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: routing to a fake bridge, not the real Rust transform"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_azure_ai_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: provider-prefix stripping before routing"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_rust_path_converts_file_document_before_bridge", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion happens in Python before the bridge call"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: Python exception-type mapping on bridge failure"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_routes_to_async_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: async routing to a fake bridge"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: async exception-type mapping on bridge failure"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_does_not_route_to_rust_when_disabled", "status": "unmapped", "reason": "bridge-plumbing: Python control flow for the toggle-disabled branch, no Rust-owned behavior runs"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_falls_back_to_python_when_bridge_unavailable", "status": "unmapped", "reason": "bridge-plumbing: Python-only fallback when the compiled Rust extension is absent, Rust cannot test its own absence"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_forwards_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site forwards a timeout kwarg, Rust receives an already-constructed request"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_passes_default_request_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site supplies a default timeout kwarg, no Rust equivalent"}, + {"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_provider_configs_expose_api_key_env_vars", "status": "unmapped", "reason": "asserts per-provider get_api_key_env_var() strings; the closest Rust test (ocr_dispatch_supports_migrated_providers) asserts provider dispatch/param resolution instead, not API key env var names"}, + + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_pdf_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection is Python-only preprocessing before the bridge call"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_png_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpeg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_gif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_webp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tiff_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_bmp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_be_case_insensitive", "status": "unmapped", "reason": "file-normalization: MIME detection case handling"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_fallback_for_unknown_extension", "status": "unmapped", "reason": "file-normalization: MIME detection fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pdf_pathlib_path_to_document_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_image_pathlib_path_to_image_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_reject_bare_str_path", "status": "unmapped", "reason": "file-normalization: arbitrary-file-read guard on bare str paths"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pathlib_path", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_explicit_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_image_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object", "status": "unmapped", "reason": "file-normalization: file-like-object conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object_with_name", "status": "unmapped", "reason": "file-normalization: file-like-object name-based MIME detection"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_missing_file_field", "status": "unmapped", "reason": "file-normalization: missing-field validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_nonexistent_pathlib_path", "status": "unmapped", "reason": "file-normalization: missing-file validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_empty_file", "status": "unmapped", "reason": "file-normalization: empty-file validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_unsupported_type", "status": "unmapped", "reason": "file-normalization: unsupported input type validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_invalid_mime_type", "status": "unmapped", "reason": "file-normalization: MIME-type injection validation"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_override_mime_type_for_pathlib_path", "status": "unmapped", "reason": "file-normalization: explicit MIME override precedence"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_document_url_for_pdf", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_png", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_jpeg", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_octet_stream", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_none", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_fallback_to_octet_stream_for_unknown", "status": "unmapped", "reason": "file-normalization: default MIME fallback"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_preserve_base64_content_correctly", "status": "unmapped", "reason": "file-normalization: binary round-trip through base64"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_from_content_type", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_with_multiple_params", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_reject_file_type_document_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body file-type guard, a different mechanism than Rust's URL-fetch SSRF guard"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_accept_document_url_type_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_raise_on_invalid_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing error path"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_ignore_document_form_field_injection", "status": "unmapped", "reason": "proxy-layer multipart form-field injection guard, a different mechanism than Rust's URL-fetch SSRF guard"}, + + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_header_in_supported_params", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "extract_header_is_a_supported_ocr_param", "justification": "both assert extract_header appears in the Mistral supported OCR params list"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_footer_in_supported_params", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "extract_footer_is_a_supported_ocr_param", "justification": "both assert extract_footer appears in the Mistral supported OCR params list"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_existing_params_still_present", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "existing_ocr_params_remain_supported", "justification": "both assert the previously supported params are still present in the supported list"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_passed_through", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_forwards_extract_header", "justification": "both assert extract_header alone survives map_ocr_params unchanged"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_footer_passed_through", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_forwards_extract_footer", "justification": "both assert extract_footer alone survives map_ocr_params unchanged"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_and_footer_together", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_forwards_extract_header_and_footer", "justification": "both assert header and footer passed together are both forwarded with their given values"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_unknown_param_is_dropped", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert an unrecognized param key is dropped while a known one is kept"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewSupportedParams::test_new_param_in_supported_list", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "new_ocr_params_are_supported", "justification": "both assert each OCR4 param is in the supported list"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewParamsMapOcr::test_new_param_passed_through", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_forwards_new_ocr_params", "justification": "both assert each OCR4 param/value pair survives map_ocr_params unchanged"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_param_included_in_request_body", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_includes_each_optional_param", "justification": "both assert each optional param value lands in the built request body alongside model/document with no files"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_multiple_new_params_together", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_includes_multiple_new_params", "justification": "both assert multiple OCR4 params passed together all land in the same request body"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_response_preserves_blocks_and_confidence_scores", "justification": "both assert blocks and confidence_scores survive the OCR response transform on the returned page"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_response_preserves_ocr4_page_fields", "justification": "both assert tables, hyperlinks, header and footer survive the OCR response transform on the returned page"}, + + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_model_info_ocr4_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr4_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_pricing_entry", "status": "unmapped", "reason": "cost-calc: cost-map JSON entry validation is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_model_info_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_ocr_and_annotation_pages_at_their_own_rates", "status": "unmapped", "reason": "cost-calc: mixed-rate billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_only_response", "status": "unmapped", "reason": "cost-calc: annotation-only billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_pages_when_pages_processed_missing", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, + {"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"}, + + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_serves_default_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_skipped_for_native_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_native_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "provider-support validation for req_format happens in Python"}, + {"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_unknown_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "req_format validation error path is Python-only"}, + + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_ocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_aocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_document_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_image_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_no_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_invalid_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_input_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_single_page", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_multiple_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_empty_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_page_with_empty_markdown", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_preserves_page_metadata", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_output_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"}, + {"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestPIIMaskingScenario::test_pii_masking_in_ocr_pages", "status": "unmapped", "reason": "PII redaction in the translation handler has no Rust equivalent"}, + + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_read_req_format_from_header", "status": "unmapped", "reason": "proxy-layer header parsing has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_prefer_body_req_format_over_header", "status": "unmapped", "reason": "proxy-layer body-vs-header precedence has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_omit_req_format_when_header_absent", "status": "unmapped", "reason": "proxy-layer parsing has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_reject_unknown_req_format", "status": "unmapped", "reason": "proxy-layer validation has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_native_payload_with_litellm_response_headers", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"}, + {"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_normalized_response_when_no_native_payload", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"} ], "rust_only_tests": [ - { - "rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", - "rust_test": "rust_custom_logger_reads_failure_payload_for_non_ocr_call_type", - "reason": "exercises the non-OCR (acompletion) call-type branch of the logger; the OCR branch is covered separately by rust_custom_logger_reads_success_payload_for_ocr" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", - "rust_test": "no_callback_fast_path_dispatches_nothing", - "reason": "Rust-only fast-path optimization test for when zero callbacks are registered; Python has no equivalent no-op dispatch path" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", - "rust_test": "with_standard_logging_payload_keeps_top_level_fields_in_sync", - "reason": "Rust-internal builder-method invariant, Python has no equivalent internal builder" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", - "rust_test": "blocks_private_and_metadata_ips", - "reason": "SSRF IP-blocking helper has no Python unit test; Python relies on the proxy-layer JSON/form guards instead" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", - "rust_test": "convert_document_url_rejects_loopback_fetch", - "reason": "URL-fetch SSRF protection is Rust-gateway-only" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", - "rust_test": "convert_document_url_leaves_data_uri_untouched", - "reason": "URL-fetch SSRF protection is Rust-gateway-only" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "truncate_error_body_passes_short_strings_through", - "reason": "Rust-gateway error-body truncation helper has no Python counterpart" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "truncate_error_body_caps_long_payloads", - "reason": "Rust-gateway error-body truncation helper has no Python counterpart" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "truncate_error_body_does_not_split_multibyte_chars", - "reason": "Rust-gateway error-body truncation helper has no Python counterpart" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "ocr_dispatch_supports_migrated_providers", - "reason": "Rust-internal provider-config dispatch table has no equivalent Python unit test" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "string_headers_accepts_string_values", - "reason": "Rust-gateway header-coercion helper has no Python counterpart" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "auth_header_detection_is_case_insensitive", - "reason": "Rust-gateway header-detection helper has no Python counterpart" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "ocr_lifecycle_runs_pre_during_and_success_hooks", - "reason": "full gateway-level guardrail-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "ocr_lifecycle_runs_failure_hook_on_provider_error", - "reason": "full gateway-level failure-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "ocr_lifecycle_pre_call_block_skips_provider_socket", - "reason": "full gateway-level pre-call-block-plus-socket-skip test with no Python equivalent at this integration scope" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "ocr_does_not_duplicate_authorization_header_when_header_is_supplied", - "reason": "outgoing HTTP header dedup at the Rust gateway has no Python counterpart" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "document_intelligence_poll_uses_resolved_subscription_key", - "reason": "full Azure DI poll-loop integration test with no Python equivalent at this scope" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", - "rust_test": "string_headers_rejects_non_string_values", - "reason": "Rust-gateway header-coercion error path has no Python counterpart" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", - "rust_test": "azure_ai_reuses_mistral_body_transform", - "reason": "Rust-internal delegation-to-Mistral-transform implementation detail, no Python test asserts this delegation" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", - "rust_test": "document_intelligence_request_uses_base64_source_for_data_uri", - "reason": "no Python test asserts on the base64Source request body shape" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", - "rust_test": "vertex_mistral_url_uses_project_location_and_model", - "reason": "vertex OCR support has no Python unit test coverage yet" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", - "rust_test": "vertex_mistral_reuses_mistral_body_transform", - "reason": "vertex OCR support has no Python unit test coverage yet" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", - "rust_test": "vertex_deepseek_request_uses_ocr_endpoint_shape", - "reason": "vertex OCR support has no Python unit test coverage yet" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", - "rust_test": "vertex_deepseek_response_wraps_markdown_content", - "reason": "vertex OCR support has no Python unit test coverage yet" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "transform_ocr_request_rejects_non_object_document", - "reason": "non-object document rejection has no dedicated Python unit test" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "transform_ocr_response_normalizes_mistral_json", - "reason": "Python's response tests target OCR4-specific fields only, none asserts the same base normalization this Rust test checks" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "complete_url_defaults_and_dedupes_v1", - "reason": "URL-building/defaulting for Mistral has no Python unit test" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "resolve_api_key_prefers_param_then_env", - "reason": "API key resolution precedence at the Rust provider-config layer has no Python unit test" - }, - { - "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", - "rust_test": "resolve_api_key_errors_when_absent", - "reason": "API key resolution error path at the Rust provider-config layer has no Python unit test" - }, - { - "rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", - "rust_test": "rust_custom_logger_reads_success_payload_for_ocr", - "reason": "Rust-internal custom-logger dispatch for OCR payloads has no Python unit test at this layer" - } + {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_maps_features", "reason": "Rust retains the feature list while filtering unsupported parameters; Python normalizes the list to a string during mapping"}, + {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_normalizes_zero_based_pages", "reason": "Python covers ascending page indices with features, but has no dedicated test for deduplicating and sorting page indices"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_failure_payload_for_non_ocr_call_type", "reason": "exercises the non-OCR (acompletion) call-type branch of the logger; the OCR branch is covered separately by rust_custom_logger_reads_success_payload_for_ocr"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "no_callback_fast_path_dispatches_nothing", "reason": "Rust-only fast-path optimization test for when zero callbacks are registered; Python has no equivalent no-op dispatch path"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "with_standard_logging_payload_keeps_top_level_fields_in_sync", "reason": "Rust-internal builder-method invariant, Python has no equivalent internal builder"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "blocks_private_and_metadata_ips", "reason": "SSRF IP-blocking helper has no Python unit test; Python relies on the proxy-layer JSON/form guards instead"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_rejects_loopback_fetch", "reason": "URL-fetch SSRF protection is Rust-gateway-only"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_leaves_data_uri_untouched", "reason": "URL-fetch SSRF protection is Rust-gateway-only"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_passes_short_strings_through", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_caps_long_payloads", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_does_not_split_multibyte_chars", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_dispatch_supports_migrated_providers", "reason": "Rust-internal provider-config dispatch table has no equivalent Python unit test"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_accepts_string_values", "reason": "Rust-gateway header-coercion helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "auth_header_detection_is_case_insensitive", "reason": "Rust-gateway header-detection helper has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_pre_during_and_success_hooks", "reason": "full gateway-level guardrail-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_failure_hook_on_provider_error", "reason": "full gateway-level failure-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_pre_call_block_skips_provider_socket", "reason": "full gateway-level pre-call-block-plus-socket-skip test with no Python equivalent at this integration scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_does_not_duplicate_authorization_header_when_header_is_supplied", "reason": "outgoing HTTP header dedup at the Rust gateway has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "document_intelligence_poll_uses_resolved_subscription_key", "reason": "full Azure DI poll-loop integration test with no Python equivalent at this scope"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_rejects_non_string_values", "reason": "Rust-gateway header-coercion error path has no Python counterpart"}, + {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "azure_ai_reuses_mistral_body_transform", "reason": "Rust-internal delegation-to-Mistral-transform implementation detail, no Python test asserts this delegation"}, + {"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_request_uses_base64_source_for_data_uri", "reason": "no Python test asserts on the base64Source request body shape"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_url_uses_project_location_and_model", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_reuses_mistral_body_transform", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_request_uses_ocr_endpoint_shape", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_response_wraps_markdown_content", "reason": "vertex OCR support has no Python unit test coverage yet"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_rejects_non_object_document", "reason": "non-object document rejection has no dedicated Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_response_normalizes_mistral_json", "reason": "Python's response tests target OCR4-specific fields only, none asserts the same base normalization this Rust test checks"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "complete_url_defaults_and_dedupes_v1", "reason": "URL-building/defaulting for Mistral has no Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_prefers_param_then_env", "reason": "API key resolution precedence at the Rust provider-config layer has no Python unit test"}, + {"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_errors_when_absent", "reason": "API key resolution error path at the Rust provider-config layer has no Python unit test"}, + {"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_success_payload_for_ocr", "reason": "Rust-internal custom-logger dispatch for OCR payloads has no Python unit test at this layer"} ] } diff --git a/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py b/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py index b8355e04266..d805311e488 100644 --- a/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py +++ b/tests/rust-python-harness/strategies/unit_tests/mapping_validator.py @@ -1,12 +1,84 @@ from __future__ import annotations +from collections import Counter +from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path +from typing import Final + +from pydantic import BaseModel, ConfigDict from ...shared.parity.ledger import TestLedger, load_ledger from .python_runner import enumerate_python_tests from .rust_runner import enumerate_rust_tests + +class TestMapping(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + python: str + rust: str + + +@dataclass(frozen=True, slots=True) +class MappingReport: + pairs: tuple[TestMapping, ...] + problems: tuple[str, ...] + + +def _name(node: str) -> str: + return node.rsplit("::", 1)[-1].split("[", 1)[0] + + +def validate_mapping( + python_tests: Sequence[str], + rust_tests: Sequence[str], + annotations: Sequence[TestMapping] = (), +) -> MappingReport: + explicit_problems: Final = ( + *(f"missing Python counterpart: {pair.python}" for pair in annotations if pair.python not in python_tests), + *(f"missing Rust counterpart: {pair.rust}" for pair in annotations if pair.rust not in rust_tests), + *( + f"ambiguous Python annotation: {name}" + for name, count in Counter(p.python for p in annotations).items() + if count > 1 + ), + *( + f"ambiguous Rust annotation: {name}" + for name, count in Counter(p.rust for p in annotations).items() + if count > 1 + ), + ) + explicit_python: Final = {pair.python for pair in annotations} + candidates: Final = { + python: tuple(rust for rust in rust_tests if _name(python) == _name(rust)) + for python in python_tests + if python not in explicit_python + } + pairs: Final = ( + *annotations, + *(TestMapping(python=python, rust=matches[0]) for python, matches in candidates.items() if len(matches) == 1), + ) + problems: Final = ( + *explicit_problems, + *(f"missing Rust counterpart: {python}" for python, matches in candidates.items() if not matches), + *( + f"ambiguous Rust counterparts: {python}: {matches}" + for python, matches in candidates.items() + if len(matches) > 1 + ), + *( + f"ambiguous Python counterparts: {rust}" + for rust, count in Counter(pair.rust for pair in pairs).items() + if count > 1 + ), + *(f"missing Python counterpart: {rust}" for rust in rust_tests if rust not in {pair.rust for pair in pairs}), + *(("no Python tests collected",) if not python_tests else ()), + *(("no Rust tests collected",) if not rust_tests else ()), + ) + return MappingReport(pairs, problems) + + REPO_ROOT = Path(__file__).resolve().parents[4] LEDGER_ROOT = Path(__file__).parent / "ledgers" diff --git a/tests/rust-python-harness/strategies/unit_tests/python_runner.py b/tests/rust-python-harness/strategies/unit_tests/python_runner.py index a7528d27756..900b30180dc 100644 --- a/tests/rust-python-harness/strategies/unit_tests/python_runner.py +++ b/tests/rust-python-harness/strategies/unit_tests/python_runner.py @@ -1,7 +1,189 @@ from __future__ import annotations +import argparse import ast +import importlib +import os +import subprocess +import sys +import tempfile +from collections.abc import Callable, Sequence from pathlib import Path +from typing import Final, Literal, cast + +import pytest +from pydantic import BaseModel, ConfigDict + +Backend = Literal["python", "rust"] + + +class PythonReport(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + backend: Backend + verified: bool + tests: tuple[str, ...] = () + outcomes: tuple[tuple[str, str, str], ...] = () + exit_code: int + problems: tuple[str, ...] = () + + +class BackendSpec(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + environment_variable: str + probe: str + + +def ocr_backend() -> Backend: + from litellm.rust_bridge import native_bridge_available + from litellm.rust_bridge.configuration import rust_ocr_enabled + + if not rust_ocr_enabled(): + return "python" + if not native_bridge_available(): + raise RuntimeError("Rust OCR was enabled but the native extension is unavailable") + return "rust" + + +class ResultPlugin: + def __init__(self, backend: Backend, probe: Callable[[], object]) -> None: + self.backend: Final = backend + self.probe: Final = probe + self.tests: tuple[str, ...] = () + self.outcomes: tuple[tuple[str, str, str], ...] = () + self.problems: tuple[str, ...] = () + + def verify(self) -> None: + if self.probe() != self.backend: + raise RuntimeError(f"backend probe did not select {self.backend}") + + def pytest_collection_finish(self, session: pytest.Session) -> None: + self.tests = tuple(item.nodeid for item in session.items) + + @pytest.hookimpl(tryfirst=True) + def pytest_runtest_call(self, item: pytest.Item) -> None: + del item + self.verify() + + def pytest_collectreport(self, report: pytest.CollectReport) -> None: + if report.failed: + self.problems = (*self.problems, str(report.longrepr)) + + def pytest_runtest_logreport(self, report: pytest.TestReport) -> None: + self.outcomes = (*self.outcomes, (report.nodeid, report.when, report.outcome)) + if report.failed: + self.problems = (*self.problems, str(report.longrepr)) + + +def run_python_tests( + selectors: Sequence[str], + repo_root: Path, + backend: Backend, + spec: BackendSpec, + pytest_args: Sequence[str] = (), +) -> PythonReport: + with tempfile.TemporaryDirectory(prefix="litellm-unit-tests-") as directory: + output: Final = Path(directory) / "report.json" + command: Final = ( + sys.executable, + "-m", + __name__, + "--backend", + backend, + "--probe", + spec.probe, + "--output", + str(output), + "--", + *selectors, + *pytest_args, + ) + env: Final = { + **os.environ, + spec.environment_variable: "1" if backend == "rust" else "0", + "PYTHONPATH": os.pathsep.join((str(repo_root), os.environ.get("PYTHONPATH", ""))), + } + try: + result: Final = subprocess.run( + command, cwd=repo_root, env=env, capture_output=True, text=True, timeout=600, check=False + ) + except (OSError, subprocess.TimeoutExpired) as error: + return PythonReport(backend=backend, verified=False, exit_code=1, problems=(str(error),)) + if not output.exists(): + return PythonReport( + backend=backend, + verified=False, + exit_code=result.returncode or 1, + problems=(result.stdout + result.stderr,), + ) + report: Final = PythonReport.model_validate_json(output.read_text()) + if report.exit_code != result.returncode: + return report.model_copy( + update={ + "exit_code": result.returncode or 1, + "problems": (*report.problems, "worker exit code differs from report"), + } + ) + return report + + +def compare_python_runs(python: PythonReport, rust: PythonReport) -> tuple[str, ...]: + return ( + *(("backend selection was not verified",) if not python.verified or not rust.verified else ()), + *(("Python run used the wrong backend",) if python.backend != "python" else ()), + *(("Rust run used the wrong backend",) if rust.backend != "rust" else ()), + *(("Python/Rust test inventories differ",) if python.tests != rust.tests else ()), + *(("Python/Rust test outcomes differ",) if sorted(python.outcomes) != sorted(rust.outcomes) else ()), + *(("no Python tests collected",) if not python.tests else ()), + *( + ("Python tests did not all pass",) + if set(python.tests) + != {node for node, phase, status in python.outcomes if phase == "call" and status == "passed"} + else () + ), + *( + ("Rust-enabled Python tests did not all pass",) + if set(rust.tests) + != {node for node, phase, status in rust.outcomes if phase == "call" and status == "passed"} + else () + ), + *(("Python test run failed",) if python.exit_code else ()), + *(("Rust-enabled Python test run failed",) if rust.exit_code else ()), + *python.problems, + *rust.problems, + ) + + +def main(argv: Sequence[str] | None = None) -> int: + parser: Final = argparse.ArgumentParser() + parser.add_argument("--backend", required=True, choices=("python", "rust")) + parser.add_argument("--probe", required=True) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("pytest_args", nargs=argparse.REMAINDER) + args: Final = parser.parse_args(argv) + try: + module, name = args.probe.rsplit(":", 1) + probe: Final = cast(Callable[[], object], getattr(importlib.import_module(module), name)) + plugin: Final = ResultPlugin(args.backend, probe) + plugin.verify() + code: Final = int( + pytest.main(["-o", "consider_namespace_packages=true", *args.pytest_args[1:]], plugins=[plugin]) + ) + report: Final = PythonReport( + backend=args.backend, + verified=True, + tests=plugin.tests, + outcomes=plugin.outcomes, + exit_code=code, + problems=plugin.problems, + ) + except Exception as error: + failure: Final = PythonReport(backend=args.backend, verified=False, exit_code=1, problems=(str(error),)) + args.output.write_text(failure.model_dump_json()) + return 1 + args.output.write_text(report.model_dump_json()) + return report.exit_code def enumerate_python_tests(repo_root: Path, relative_path: str) -> frozenset[str]: @@ -20,3 +202,7 @@ def enumerate_python_tests(repo_root: Path, relative_path: str) -> frozenset[str module_level.append(f"{node.name}::{child.name}") return frozenset(module_level) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/rust-python-harness/strategies/unit_tests/runner.py b/tests/rust-python-harness/strategies/unit_tests/runner.py new file mode 100644 index 00000000000..d10fa364d1c --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/runner.py @@ -0,0 +1,91 @@ +from __future__ import annotations + +import sys +from collections.abc import Sequence +from pathlib import Path +from time import monotonic +from typing import Final + +from pydantic import BaseModel, ConfigDict + +from ...shared.reporting.models import HarnessCase, HarnessRun, RunStatus +from ...shared.reporting.pytest_runner import UpdateCallback +from .mapping_validator import TestMapping, validate_mapping +from .python_runner import BackendSpec, compare_python_runs, run_python_tests +from .rust_runner import run_rust_tests + + +class UnitSuite(BaseModel): + model_config = ConfigDict(frozen=True, extra="forbid") + + python_selectors: tuple[str, ...] + cargo_manifest: str + cargo_package: str + cargo_filter: str + backend: BackendSpec + mappings: tuple[TestMapping, ...] = () + + +def run_suite(suite: UnitSuite, repo_root: Path, pytest_args: Sequence[str] = ()) -> tuple[str, ...]: + if not suite.python_selectors or not suite.cargo_filter: + return ("unit suites must select Python tests and a focused Cargo filter",) + python: Final = run_python_tests(suite.python_selectors, repo_root, "python", suite.backend, pytest_args) + rust_python: Final = run_python_tests(suite.python_selectors, repo_root, "rust", suite.backend, pytest_args) + inventory: Final = run_rust_tests( + repo_root / suite.cargo_manifest, suite.cargo_package, suite.cargo_filter, collect_only=True + ) + mapping: Final = validate_mapping(python.tests, inventory.tests, suite.mappings) + rust: Final = run_rust_tests(repo_root / suite.cargo_manifest, suite.cargo_package, suite.cargo_filter) + return ( + *compare_python_runs(python, rust_python), + *mapping.problems, + *(("native Rust tests did not all pass",) if set(inventory.tests) != set(rust.tests) else ()), + *((inventory.output,) if inventory.exit_code else ()), + *((rust.output,) if rust.exit_code else ()), + ) + + +def run( + cases: Sequence[HarnessCase], + repo_root: Path, + on_update: UpdateCallback, + pytest_args: Sequence[str] = (), +) -> tuple[int, HarnessRun]: + report: Final = HarnessRun.from_cases(cases) + for case in cases: + result: Final = report.results[case.key] + if case.unit_suite is None: + result.finalize() + continue + nodeid: Final = f"unit-suite:{case.unit_suite}" + result.collected.add(nodeid) + result.status = RunStatus.RUNNING + on_update(report) + try: + suite: Final = UnitSuite.model_validate_json((repo_root / case.unit_suite).read_text()) + problems: Final = run_suite(suite, repo_root, pytest_args) + except (OSError, ValueError) as error: + result.record(nodeid, RunStatus.ERROR) + report.failures.append((nodeid, str(error))) + continue + result.record(nodeid, RunStatus.FAILED if problems else RunStatus.PASSED) + report.failures.extend((nodeid, problem) for problem in problems) + on_update(report) + report.finished_at = monotonic() + on_update(report) + return int( + any( + result.status in {RunStatus.ERROR, RunStatus.FAILED, RunStatus.MISSING} + for result in report.results.values() + ) + ), report + + +def main(argv: Sequence[str] | None = None) -> int: + from ...cli import main as harness_main + + return harness_main(argv, strategy_id="unit_tests") + + +if __name__ == "__main__": + raise SystemExit(main(sys.argv[1:])) diff --git a/tests/rust-python-harness/strategies/unit_tests/rust_runner.py b/tests/rust-python-harness/strategies/unit_tests/rust_runner.py index 6b855adbc4b..b24034199b5 100644 --- a/tests/rust-python-harness/strategies/unit_tests/rust_runner.py +++ b/tests/rust-python-harness/strategies/unit_tests/rust_runner.py @@ -1,7 +1,47 @@ from __future__ import annotations import re +import subprocess +from dataclasses import dataclass from pathlib import Path +from typing import Final + + +@dataclass(frozen=True, slots=True) +class RustReport: + tests: tuple[str, ...] + exit_code: int + output: str + + +def run_rust_tests(manifest: Path, package: str, test_filter: str, *, collect_only: bool = False) -> RustReport: + command: Final = ( + "cargo", + "test", + "--manifest-path", + str(manifest), + "--package", + package, + "--lib", + test_filter, + "--", + *(("--list",) if collect_only else ("--format=pretty",)), + ) + try: + result: Final = subprocess.run(command, capture_output=True, text=True, check=False, timeout=600) + except (OSError, subprocess.TimeoutExpired) as error: + return RustReport((), 1, str(error)) + tests: Final = ( + tuple(line.removesuffix(": test") for line in result.stdout.splitlines() if line.endswith(": test")) + if collect_only + else tuple( + line.removeprefix("test ").removesuffix(" ... ok") + for line in result.stdout.splitlines() + if line.startswith("test ") and line.endswith(" ... ok") + ) + ) + return RustReport(tests, result.returncode, result.stdout + result.stderr) + _RUST_TEST_PATTERN = re.compile( r"#\[(?:test|tokio::test)\][^\n]*\n(?:[^\n]*\n)*?\s*(?:async\s+)?fn\s+(\w+)\s*\(" diff --git a/tests/rust-python-harness/strategies/unit_tests/strategy.json b/tests/rust-python-harness/strategies/unit_tests/strategy.json new file mode 100644 index 00000000000..7ae5d9c22c6 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/strategy.json @@ -0,0 +1,32 @@ +{ + "order": 30, + "id": "unit_tests", + "label": "Unit tests", + "description": "Validate Python/Rust test mappings and compare isolated Python runs alongside native Cargo tests.", + "functions": { + "ocr": { + "coverage": "planned", + "selectors": [] + }, + "messages": { + "coverage": "planned", + "selectors": [] + }, + "chat_completions": { + "coverage": "planned", + "selectors": [] + }, + "responses": { + "coverage": "planned", + "selectors": [] + }, + "count_tokens": { + "coverage": "planned", + "selectors": [] + }, + "transcription": { + "coverage": "planned", + "selectors": [] + } + } +} diff --git a/tests/rust-python-harness/strategies/unit_tests/test_mapping_validator.py b/tests/rust-python-harness/strategies/unit_tests/test_mapping_validator.py new file mode 100644 index 00000000000..25c63faf3a7 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/test_mapping_validator.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import pytest + +from .mapping_validator import TestMapping as Mapping, validate_mapping + + +def test_matches_names_and_explicit_annotations() -> None: + report = validate_mapping( + ("tests/test_api.py::test_decode", "tests/test_api.py::test_error"), + ("api::test_decode", "api::preserves_error"), + (Mapping(python="tests/test_api.py::test_error", rust="api::preserves_error"),), + ) + assert report.problems == () + assert {(pair.python, pair.rust) for pair in report.pairs} == { + ("tests/test_api.py::test_decode", "api::test_decode"), + ("tests/test_api.py::test_error", "api::preserves_error"), + } + + +@pytest.mark.parametrize( + ("python", "rust", "message"), + ( + (("test_decode",), (), "missing Rust counterpart"), + ((), ("test_decode",), "missing Python counterpart"), + (("test_decode",), ("one::test_decode", "two::test_decode"), "ambiguous Rust counterparts"), + (("one::test_decode", "two::test_decode"), ("test_decode",), "ambiguous Python counterparts"), + ), +) +def test_reports_missing_and_ambiguous_counterparts( + python: tuple[str, ...], rust: tuple[str, ...], message: str +) -> None: + assert any(message in problem for problem in validate_mapping(python, rust).problems) + + +def test_rejects_stale_annotations_even_when_names_match() -> None: + report = validate_mapping(("test_decode",), ("test_decode",), (Mapping(python="test_decode", rust="removed"),)) + assert "missing Rust counterpart: removed" in report.problems diff --git a/tests/rust-python-harness/strategies/unit_tests/test_python_runner.py b/tests/rust-python-harness/strategies/unit_tests/test_python_runner.py new file mode 100644 index 00000000000..ed4920b7525 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/test_python_runner.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import os +from pathlib import Path +from typing import Final + +from .python_runner import BackendSpec, compare_python_runs, run_python_tests + +HARNESS_ROOT: Final = Path(__file__).resolve().parents[4] + + +def _suite(root: Path, *, mismatch: bool = False) -> BackendSpec: + (root / "pytest.ini").write_text("[pytest]\n") + (root / "backend_probe.py").write_text( + "import os\ndef selected():\n return 'rust' if os.environ.get('TEST_USE_RUST') == '1' else 'python'\n" + ) + (root / "test_backend.py").write_text( + "import os\nfrom pathlib import Path\nfrom backend_probe import selected\n" + "def test_backend():\n backend = selected()\n" + " Path(backend + '.pid').write_text(str(os.getpid()))\n" + + (" assert backend == 'python'\n" if mismatch else " assert backend in {'python', 'rust'}\n") + ) + return BackendSpec(environment_variable="TEST_USE_RUST", probe="backend_probe:selected") + + +def test_runs_existing_python_tests_in_separate_verified_backends(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("PYTHONPATH", str(HARNESS_ROOT)) + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + spec: Final = _suite(tmp_path) + python: Final = run_python_tests(("test_backend.py",), tmp_path, "python", spec) + rust: Final = run_python_tests(("test_backend.py",), tmp_path, "rust", spec) + assert compare_python_runs(python, rust) == () + assert python.tests == ("test_backend.py::test_backend",) + assert (tmp_path / "python.pid").read_text() != (tmp_path / "rust.pid").read_text() + assert (tmp_path / "python.pid").read_text() != str(os.getpid()) + + +def test_rejects_wrong_backend_and_different_test_results(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("PYTHONPATH", str(HARNESS_ROOT)) + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + spec: Final = _suite(tmp_path, mismatch=True) + python: Final = run_python_tests(("test_backend.py",), tmp_path, "python", spec) + rust: Final = run_python_tests(("test_backend.py",), tmp_path, "rust", spec) + assert "Python/Rust test outcomes differ" in compare_python_runs(python, rust) + wrong: Final = run_python_tests( + ("test_backend.py",), + tmp_path, + "rust", + BackendSpec(environment_variable="WRONG_FLAG", probe=spec.probe), + ) + assert wrong.exit_code == 1 + assert not wrong.verified + assert "backend probe did not select rust" in wrong.problems[0] diff --git a/tests/rust-python-harness/strategies/unit_tests/test_runner.py b/tests/rust-python-harness/strategies/unit_tests/test_runner.py new file mode 100644 index 00000000000..c652d6e12b1 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/test_runner.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Final + +import pytest + +from ...shared.reporting.models import Coverage, HarnessCase, RunStatus +from .runner import run + + +@pytest.mark.skipif(shutil.which("cargo") is None, reason="Cargo is required for the combined unit strategy") +def test_combines_mapping_backend_comparison_and_cargo_results(tmp_path: Path, monkeypatch) -> None: + monkeypatch.setenv("PYTHONPATH", str(Path(__file__).resolve().parents[4])) + monkeypatch.setenv("PYTEST_DISABLE_PLUGIN_AUTOLOAD", "1") + (tmp_path / "pytest.ini").write_text("[pytest]\n") + (tmp_path / "backend_probe.py").write_text( + "import os\ndef selected():\n return 'rust' if os.environ['TEST_USE_RUST'] == '1' else 'python'\n" + ) + (tmp_path / "test_api.py").write_text("def test_decode():\n assert int('42') == 42\n") + (tmp_path / "Cargo.toml").write_text( + '[package]\nname = "combined-check"\nversion = "0.1.0"\nedition = "2021"\n[workspace]\n' + ) + (tmp_path / "src").mkdir() + (tmp_path / "src/lib.rs").write_text('#[test] fn test_decode() { assert_eq!("42".parse::().unwrap(), 42); }\n') + suite: Final = { + "python_selectors": ("test_api.py",), + "cargo_manifest": "Cargo.toml", + "cargo_package": "combined-check", + "cargo_filter": "test_decode", + "backend": {"environment_variable": "TEST_USE_RUST", "probe": "backend_probe:selected"}, + } + (tmp_path / "suite.json").write_text(json.dumps(suite)) + case: Final = HarnessCase( + strategy_id="unit_tests", + strategy_label="Unit tests", + sdk_function="ocr", + coverage=Coverage.COMPLETE, + selectors=(), + unit_suite="suite.json", + ) + code, report = run((case,), tmp_path, lambda _: None) + assert code == 0, report.failures + assert report.results[case.key].status is RunStatus.PASSED + (tmp_path / "suite.json").write_text( + json.dumps({**suite, "mappings": [{"python": "test_api.py::test_decode", "rust": "removed"}]}) + ) + failed_code, failed_report = run((case,), tmp_path, lambda _: None) + assert failed_code == 1 + assert failed_report.results[case.key].status is RunStatus.FAILED + assert any("missing Rust counterpart: removed" in detail for _, detail in failed_report.failures) + + (tmp_path / "suite.json").write_text(json.dumps(suite)) + (tmp_path / "src/lib.rs").write_text("#[test] #[ignore] fn test_decode() {}\n") + skipped_code, skipped_report = run((case,), tmp_path, lambda _: None) + assert skipped_code == 1 + assert any("native Rust tests did not all pass" in detail for _, detail in skipped_report.failures) diff --git a/tests/rust-python-harness/strategies/unit_tests/test_rust_runner.py b/tests/rust-python-harness/strategies/unit_tests/test_rust_runner.py new file mode 100644 index 00000000000..aeeb7f602f5 --- /dev/null +++ b/tests/rust-python-harness/strategies/unit_tests/test_rust_runner.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import shutil +from pathlib import Path +from typing import Final + +import pytest + +from .rust_runner import run_rust_tests + + +@pytest.mark.skipif(shutil.which("cargo") is None, reason="Cargo is required for native runner integration") +def test_collects_and_runs_native_tests_and_propagates_failure(tmp_path: Path) -> None: + manifest: Final = tmp_path / "Cargo.toml" + manifest.write_text('[package]\nname = "harness-runner-check"\nversion = "0.1.0"\nedition = "2021"\n[workspace]\n') + (tmp_path / "src").mkdir() + source: Final = tmp_path / "src/lib.rs" + source.write_text("#[test] fn test_parity() { assert_eq!(2 + 2, 4); }\n") + inventory: Final = run_rust_tests(manifest, "harness-runner-check", "test_parity", collect_only=True) + assert inventory.exit_code == 0, inventory.output + assert inventory.tests == ("test_parity",) + passing: Final = run_rust_tests(manifest, "harness-runner-check", "test_parity") + assert passing.exit_code == 0, passing.output + source.write_text("#[test] fn test_parity() { assert_eq!(2 + 2, 5); }\n") + failed: Final = run_rust_tests(manifest, "harness-runner-check", "test_parity") + assert failed.exit_code != 0 + assert "test_parity" in failed.output diff --git a/tests/rust-python-harness/unit_tests_rust/README.md b/tests/rust-python-harness/unit_tests_rust/README.md deleted file mode 100644 index 12c7eb0089c..00000000000 --- a/tests/rust-python-harness/unit_tests_rust/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Rust unit tests - -Holds focused Cargo tests for Rust-owned parsing, transforms, errors, and streaming behavior. These tests make failures fast to diagnose before the Python bridge or full SDK path is involved. diff --git a/tests/rust-python-harness/unit_tests_rust/strategy.json b/tests/rust-python-harness/unit_tests_rust/strategy.json deleted file mode 100644 index bfb2bc0dad0..00000000000 --- a/tests/rust-python-harness/unit_tests_rust/strategy.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "order": 20, - "id": "unit_tests_rust", - "label": "Rust unit tests", - "description": "Exercise Rust-owned behavior directly with focused unit tests.", - "functions": { - "ocr": {"coverage": "planned", "selectors": []}, - "messages": {"coverage": "planned", "selectors": []}, - "responses": {"coverage": "planned", "selectors": []}, - "count_tokens": {"coverage": "planned", "selectors": []}, - "chat_completions": {"coverage": "planned", "selectors": []}, - "transcription": {"coverage": "planned", "selectors": []} - } -} diff --git a/tests/rust-python-harness/validate_sub_methods/README.md b/tests/rust-python-harness/validate_sub_methods/README.md deleted file mode 100644 index 24894366f23..00000000000 --- a/tests/rust-python-harness/validate_sub_methods/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Validate sub-methods - -Checks each request, response, stream, and error-mapping sub-method independently across Python and Rust. It also validates that traced Python helpers have an explicit Rust implementation and parity test. diff --git a/tests/rust-python-harness/validate_sub_methods/strategy.json b/tests/rust-python-harness/validate_sub_methods/strategy.json deleted file mode 100644 index a26bc2d70ed..00000000000 --- a/tests/rust-python-harness/validate_sub_methods/strategy.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "order": 30, - "id": "validate_sub_methods", - "label": "Validate sub-methods", - "description": "Compare isolated transforms and verify Python-to-Rust helper coverage.", - "functions": { - "ocr": {"coverage": "planned", "selectors": []}, - "messages": {"coverage": "planned", "selectors": []}, - "responses": {"coverage": "planned", "selectors": []}, - "count_tokens": {"coverage": "planned", "selectors": []}, - "chat_completions": {"coverage": "planned", "selectors": []}, - "transcription": {"coverage": "planned", "selectors": []} - } -} diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py index 0ce49d51aed..a89a985952b 100644 --- a/tests/test_litellm/rust_bridge/native_route_wheel_test.py +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -200,7 +200,7 @@ def success_value(route: str, response: dict[object, object]) -> object: def assert_rate_limit(native: object, route: str, error: BaseException) -> None: - if route == "chat_completions": + if route in {"ocr", "chat_completions"}: upstream_error: Final = native.RustUpstreamError if not isinstance(error, upstream_error) or error.args[0] != 429: raise AssertionError(f"{route} returned the wrong 429 error: {error!r}") diff --git a/tests/test_rust_python_harness.py b/tests/test_rust_python_harness.py index 514446577fd..a2b9c8e2a76 100644 --- a/tests/test_rust_python_harness.py +++ b/tests/test_rust_python_harness.py @@ -2,19 +2,23 @@ from __future__ import annotations import importlib import json +import os +import subprocess +import sys from pathlib import Path +from typing import Final import pytest catalog = importlib.import_module("tests.rust-python-harness.catalog") cli = importlib.import_module("tests.rust-python-harness.cli") +models = importlib.import_module("tests.rust-python-harness.shared.reporting.models") +runner = importlib.import_module("tests.rust-python-harness.shared.reporting.pytest_runner") +ui = importlib.import_module("tests.rust-python-harness.shared.reporting.ui") ledger_module = importlib.import_module("tests.rust-python-harness.shared.parity.ledger") mapping_validator = importlib.import_module( "tests.rust-python-harness.strategies.unit_tests.mapping_validator" ) -models = importlib.import_module("tests.rust-python-harness.models") -runner = importlib.import_module("tests.rust-python-harness.runner") -ui = importlib.import_module("tests.rust-python-harness.ui") load_catalog = catalog.load_catalog load_ledger = ledger_module.load_ledger @@ -70,9 +74,9 @@ def test_should_load_the_four_harness_strategies_in_order() -> None: strategies = load_catalog() assert [strategy.id for strategy in strategies] == [ - "e2e_fuzz_tests", - "unit_tests_rust", - "validate_sub_methods", + "e2e_parity", + "trace_parity", + "unit_tests", "existing_e2e_test_sdk", ] assert all( @@ -155,6 +159,46 @@ def test_should_treat_an_all_planned_filtered_run_as_success(tmp_path: Path) -> assert next(iter(run.results.values())).status is RunStatus.PLANNED +@pytest.mark.parametrize("strategy_id", ("e2e_parity", "existing_e2e_test_sdk")) +def test_should_run_namespace_package_relative_imports(tmp_path: Path, strategy_id: str) -> None: + package: Final = tmp_path / "manual_suite" / "relative-tests" + package.mkdir(parents=True) + (package / "__init__.py").write_text("", encoding="utf-8") + (package / "values.py").write_text("ANSWER = 42\n", encoding="utf-8") + (package / "test_relative.py").write_text( + "from .values import ANSWER\n\ndef test_answer():\n assert ANSWER == 42\n", + encoding="utf-8", + ) + result: Final = subprocess.run( + ( + sys.executable, + "-c", + "import importlib\n" + "from pathlib import Path\n" + "cli = importlib.import_module('tests.rust-python-harness.cli')\n" + "models = importlib.import_module('tests.rust-python-harness.shared.reporting.models')\n" + f"case = models.HarnessCase(strategy_id={strategy_id!r}, strategy_label='Example', " + "sdk_function='ocr', coverage=models.Coverage.COMPLETE, " + "selectors=('manual_suite/relative-tests/',))\n" + f"code, run = cli._resolve_runner({strategy_id!r})((case,), Path.cwd(), lambda _: None)\n" + "assert code == 0, code\n" + "assert next(iter(run.results.values())).passed == 1\n", + ), + cwd=tmp_path, + env={ + **os.environ, + "PYTHONPATH": os.pathsep.join((str(tmp_path), str(Path(__file__).resolve().parents[1]))), + "PYTEST_DISABLE_PLUGIN_AUTOLOAD": "1", + }, + capture_output=True, + text=True, + timeout=30, + check=False, + ) + + assert result.returncode == 0, result.stdout + result.stderr + + def test_should_finalize_a_fully_passing_case() -> None: result = CaseResult(case=_case(selectors=("tests/test_parity.py",))) result.set_initial_status() @@ -184,10 +228,10 @@ def test_should_replace_a_pass_with_a_teardown_error() -> None: def test_should_filter_the_catalog_by_strategy_and_sdk_function() -> None: strategies = load_catalog() - cases = _select(strategies, {"e2e_fuzz_tests"}, {"messages"}) + cases = _select(strategies, {"e2e_parity"}, {"messages"}) assert len(cases) == 1 - assert cases[0].key == "e2e_fuzz_tests:messages" + assert cases[0].key == "e2e_parity:messages" def test_should_reject_an_unknown_strategy() -> None: @@ -216,10 +260,10 @@ def test_should_format_developer_facing_run_context() -> None: assert _summary(run) == (1, 0, 0, 0) assert _format_duration(1.25) == "1.2s" assert _rerun_command("tests/test_parity.py::test_one") == ( - "poetry run pytest tests/test_parity.py::test_one -q" + "poetry run pytest tests/test_parity.py::test_one -q -o consider_namespace_packages=true" ) assert _rerun_command("tests/test_parity.py::test_one[value with spaces]") == ( - "poetry run pytest 'tests/test_parity.py::test_one[value with spaces]' -q" + "poetry run pytest 'tests/test_parity.py::test_one[value with spaces]' -q -o consider_namespace_packages=true" ) @@ -240,7 +284,7 @@ def test_should_report_confidence_for_each_sdk_section() -> None: strategies = load_catalog() cases = tuple(case for strategy in strategies for case in strategy.cases) run = HarnessRun.from_cases(cases) - passing = run.results["e2e_fuzz_tests:responses"] + passing = run.results["e2e_parity:responses"] passing.collected.add("tests/test_parity.py::test_one") passing.record("tests/test_parity.py::test_one", RunStatus.PASSED) @@ -287,6 +331,21 @@ def test_should_scope_validate_ledger_to_the_requested_function( assert "ocr" not in captured.out +@pytest.mark.parametrize("strategy_id", (None, "e2e_parity", "trace_parity", "unit_tests", "existing_e2e_test_sdk")) +def test_should_validate_chat_completions_ledger_from_each_runner( + strategy_id: str | None, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code: Final = cli.main( + ("--validate-ledger", "--function", "chat_completions"), strategy_id=strategy_id + ) + + captured: Final = capsys.readouterr() + assert exit_code == 0 + assert "chat_completions" in captured.out + assert "no ledger yet" in captured.out + assert "ocr" not in captured.out + + def test_should_have_every_python_and_rust_ocr_test_accounted_for_in_the_ledger() -> None: ledger = load_ledger(ledger_path_for("ocr")) diff --git a/uv.lock b/uv.lock index aa59ff7b229..de3181edd5a 100644 --- a/uv.lock +++ b/uv.lock @@ -10,7 +10,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-08-30T17:51:25.171404Z" +exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. exclude-newer-span = "P3D" [manifest] @@ -2373,14 +2373,14 @@ wheels = [ [[package]] name = "gitpython" -version = "3.1.58" +version = "3.1.61" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "gitdb" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/26/d6/5f358ff283325580c2003a6d953aea18cfe10ae87b46f5ebc80fa3a386dc/gitpython-3.1.58.tar.gz", hash = "sha256:621416df10ef3fd0e19fabf9172ddeed0fa704d353d04f194eec56a625a95b22", size = 228498, upload-time = "2026-08-04T15:05:49.47Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/61/3285044215fb596bf093e39ccb96ece0a1076a8ca57a61e069a6a33cdb1b/gitpython-3.1.61.tar.gz", hash = "sha256:f51c24d8c0f733a195447385f5774a5dfe8767f5acfd7994a33755644c6ecc95", size = 231680, upload-time = "2026-08-28T11:01:13.761Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/0c/9d8752098bc442f0726e64aa6135940b3a96809915d1aa4206c1bb97881d/gitpython-3.1.58-py3-none-any.whl", hash = "sha256:d331e722577f0fd7fc1f857419b3ecc07af66282b933d2a4d95f84a042fdd50f", size = 220183, upload-time = "2026-08-04T15:05:48.025Z" }, + { url = "https://files.pythonhosted.org/packages/6f/5e/49cc172da4d0578644ba37cec5cb365b1fefc603b26edea9bcac1c7f830a/gitpython-3.1.61-py3-none-any.whl", hash = "sha256:8ab28c9da863cdd9e7d7694ec46cf3e6c9a12d8a30a1acd3447aec11975d530c", size = 222118, upload-time = "2026-08-28T11:01:12.262Z" }, ] [[package]] @@ -3373,6 +3373,98 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, ] +[[package]] +name = "hypothesis" +version = "6.165.10" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "sortedcontainers" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5c/e2/0fad246d2b6330e1f78479bfc566b5c22be82aee8a865cde9a08f648487d/hypothesis-6.165.10.tar.gz", hash = "sha256:68b45e09834cd80523cb1eb274463073c7a9af4e4ef7cff34d9615f355572d32", size = 503703, upload-time = "2026-08-16T22:56:15.404Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/c1/9a9538e6d185baf5cc7f15bc3b76e08efbb3de4b3c782f234356449c0dd7/hypothesis-6.165.10-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:f839d29d0cc12048cf073d88ca4fdf94d420bc2b8afd69641ff6d496422ccd4f", size = 783243, upload-time = "2026-08-16T22:55:44.058Z" }, + { url = "https://files.pythonhosted.org/packages/a1/30/b70d9d79e871a75cbdeccd9067f20ecdb9eb2a1dfa03c630be3ad13b8b30/hypothesis-6.165.10-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:e10858f57ed0e74baa04393845f469fe8ad502c16ece4499bef7700c575611bd", size = 778815, upload-time = "2026-08-16T22:55:46.948Z" }, + { url = "https://files.pythonhosted.org/packages/db/52/6f0a9b7aab24b0635e2238f3fbddea5b54b17879ac813df42a3cc3384c5c/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76a7be86d986223b9f1bdb7e7cbcdb048649901fdb956c598ef73bdab1786cd5", size = 1108009, upload-time = "2026-08-16T22:54:53.082Z" }, + { url = "https://files.pythonhosted.org/packages/f6/06/8d0d4e11ff02350d09ec9f9e90af354158e59e16a8907ba5199a4ff2d7e8/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:717aea574e0e5edba2868aa66b1caae335d8f1ad3fb29f01dd6502953fa823a1", size = 1136596, upload-time = "2026-08-16T22:54:54.443Z" }, + { url = "https://files.pythonhosted.org/packages/59/dd/01a1e440f2e38dc1ccf5d597af5b8a0bee5f21b674c99c123b5554de9690/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4334058033e0214475f019e15492a50f3854fe8728cf51fe25c6191a2c3f8e52", size = 1135234, upload-time = "2026-08-16T22:55:08.911Z" }, + { url = "https://files.pythonhosted.org/packages/7d/18/8a26c24d3d9db20265f39df341ab265858c094e209571e3179cf237935f4/hypothesis-6.165.10-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2abb50cf1cf77d721de0a24c3f99d9c4ffdeb2cbd1e12aebb5a7a93e2b6b6d1f", size = 1157528, upload-time = "2026-08-16T22:56:02.159Z" }, + { url = "https://files.pythonhosted.org/packages/ea/8e/ce3c829b1937402d7944420ca26a05a0c8563e894dcff03d34ffa279d306/hypothesis-6.165.10-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:3de69aa8b924b400291a3cc42aaf78e6ab65c905a3e7e1a5dc39d95ef1b428cb", size = 1112870, upload-time = "2026-08-16T22:54:55.919Z" }, + { url = "https://files.pythonhosted.org/packages/f2/1b/4c4926d6c9a2b5d7cc090cc1e91219d6796102aa2a2c4b8f961c939e60b5/hypothesis-6.165.10-cp310-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5841331c504e02d7c334591681cb8587cdd59dee7e149db6d3db8e3f9e9f02eb", size = 1149683, upload-time = "2026-08-16T22:55:30.567Z" }, + { url = "https://files.pythonhosted.org/packages/cb/f9/df24eb28412f82465e2b7707f0ff1ec274d580bce389d4d9156617dc7bba/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2d0e0f8263d34dd8fa3b39eaa9a50bba56a8470b3dd9ebf6672d10840abe063e", size = 1283402, upload-time = "2026-08-16T22:54:18.054Z" }, + { url = "https://files.pythonhosted.org/packages/4d/07/c2b2a761300cf60b90ccebba4328175331e67d34f4fbd39429a7ddcdce49/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0c4e6869817c3cfdf5a2b4d348497b95159bdecb3365be732c9b8570e36a4eef", size = 1409948, upload-time = "2026-08-16T22:54:22.343Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ec/1c2bf1acdd0e273d81f833f85caf0ae5423db68a783554992fca36e6c541/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9f07ae36c3b093e13687a894e79fe69e98a94c0b67fef656c575247682218143", size = 1265023, upload-time = "2026-08-16T22:54:41.402Z" }, + { url = "https://files.pythonhosted.org/packages/3d/a8/7f984908b7391160c7801b84e51ca8e4ba88c89e8d8811aa1aa7c03de73c/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:aff1f584c9538e8979cd180b1d70bf99bc16be19d4666414f49e5942b21a4f2c", size = 1282698, upload-time = "2026-08-16T22:56:06.998Z" }, + { url = "https://files.pythonhosted.org/packages/48/78/3a5d91c2d0250521736c42dfa2402b75049bc5fe2fb716c10bc84bb91ed1/hypothesis-6.165.10-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1f2c4db25fb8ec1a16a8dba580666337b8ffb1887c4cf1750cc954313897cef7", size = 1324816, upload-time = "2026-08-16T22:54:46.675Z" }, + { url = "https://files.pythonhosted.org/packages/6f/99/27450763853a034bca1574d3e0a315164b33ff49c3862df6872dda45e25e/hypothesis-6.165.10-cp310-abi3-win32.whl", hash = "sha256:b33dc30170a7402e03c180f2c5ef69dc077152f35b91621e9cebcde9c7d71746", size = 669039, upload-time = "2026-08-16T22:55:11.962Z" }, + { url = "https://files.pythonhosted.org/packages/2c/fc/ff2988b72b5705ad9ca500444bf3f43e3c2f41edfa034bbfeb23b215791a/hypothesis-6.165.10-cp310-abi3-win_amd64.whl", hash = "sha256:e9f924aa610c0618445e1e8738c822c3190ce2a2699a0cb48ec3a351a96761f2", size = 675213, upload-time = "2026-08-16T22:55:01.697Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8b/821810d36f78d9d9421cd2c5d9d36983b45bb3575c3086276cc5c76f9f73/hypothesis-6.165.10-cp310-abi3-win_arm64.whl", hash = "sha256:1d305448e9bd8e2f4f3cea0eafd809efdaab4e998a0019bc615650c8463e42f1", size = 673537, upload-time = "2026-08-16T22:54:47.898Z" }, + { url = "https://files.pythonhosted.org/packages/26/61/5e89268ce03317fb9f82449a1b3efd9e599dee090288fd0cf7586c532fb1/hypothesis-6.165.10-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:73e6df02a6a62f8045b511c272f894d08e56d174504c793c9effcbc6778051a8", size = 783959, upload-time = "2026-08-16T22:55:29.078Z" }, + { url = "https://files.pythonhosted.org/packages/e1/e9/f4e0832e81bb53b70cf1712e28c867db64245b32595b594217452e7dbd8d/hypothesis-6.165.10-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8b20f44773a9ab84400465e318712d8c2ca16418d35b9f80aa27fdf2d690ad10", size = 779684, upload-time = "2026-08-16T22:54:57.698Z" }, + { url = "https://files.pythonhosted.org/packages/77/de/ea072d3359d5678771bed407f80439e8ac7ca905d1031b0372f61bf5746e/hypothesis-6.165.10-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb8c7d05ea27a093a92b250904095d71d924b6b44e5795a415c1b20c265f0c65", size = 1108540, upload-time = "2026-08-16T22:55:03.282Z" }, + { url = "https://files.pythonhosted.org/packages/28/56/e7c395cdaa3d6c28b944c1c3c516dee50d2b7b3aeafa31874b57009ca51f/hypothesis-6.165.10-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4dafd6d6ababfa3b14dd6e5f0378cb7c7d291895a31a40abcbb7cc74f396131", size = 1158089, upload-time = "2026-08-16T22:54:36.205Z" }, + { url = "https://files.pythonhosted.org/packages/b1/49/1c6d2c465b9c5fc3213f1be89be95ba53819ca0130248c484129ccfefb71/hypothesis-6.165.10-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fa74636a49fc8077413ce8db3e85f1c4aff880788bb55bda56253118e036fe5b", size = 1284125, upload-time = "2026-08-16T22:54:37.727Z" }, + { url = "https://files.pythonhosted.org/packages/7d/bc/7caf5ac3d0173bd57bd2a5ab854ca49a3664a4309257be1452f81025cc24/hypothesis-6.165.10-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2b112768cfb67f2b683e53e58c1a33d27811aacf60c942b8eb74635e469a73f6", size = 1325082, upload-time = "2026-08-16T22:54:38.952Z" }, + { url = "https://files.pythonhosted.org/packages/70/99/9d844330f570d6a4f127a683eab1e78c8263e6e72b16189f3534fa6bf6de/hypothesis-6.165.10-cp310-cp310-win_amd64.whl", hash = "sha256:56cb8c9055e50545fe6e3e5a560ec25a724673b2e4051f3c24d44e3ebc35dd72", size = 675082, upload-time = "2026-08-16T22:54:29.872Z" }, + { url = "https://files.pythonhosted.org/packages/ed/c2/b9546ace11f241c9c02d389f258cb80c14447a8c885771c9f1f0bc1d85ca/hypothesis-6.165.10-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:592107a0faf6c9c3a63a8dbf13dfb1cbda1cf599b0bc11c953221b00204b9ce1", size = 783716, upload-time = "2026-08-16T22:55:36.624Z" }, + { url = "https://files.pythonhosted.org/packages/37/10/27c2fdd574fd798caf5e91eb51f7834b098f5d840ce733efb3fba79ef86e/hypothesis-6.165.10-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f9180c362bde06fd05380298ded4e234fbc0d6ede0a864835bfd91c1e24283d5", size = 779507, upload-time = "2026-08-16T22:55:07.633Z" }, + { url = "https://files.pythonhosted.org/packages/5e/b6/70bc23695f3783c4b0486b6cad47b08a20f791db4a3c1b25250add9659fa/hypothesis-6.165.10-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d623801ae3dcd97b77b983400ef3d48bf976648e4efff19929175322eaae074d", size = 1108406, upload-time = "2026-08-16T22:55:39.653Z" }, + { url = "https://files.pythonhosted.org/packages/71/4c/32e200bd7a352af4b7f4e3729aaa4cd002cb5fe8c4c6aef5599d0019f152/hypothesis-6.165.10-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:20f6236cfb90b7817bb1a6a087589ca4aa46d73170f0dd62963952ed5dadc589", size = 1157850, upload-time = "2026-08-16T22:55:24.394Z" }, + { url = "https://files.pythonhosted.org/packages/03/a5/8efc2a9a484822efc0d0da466f50094e0f2c068187faaf33831fc905873e/hypothesis-6.165.10-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ad0764730e8e3421601c2cc7e1f054a9206c60ea0917165d8d9193dc453f34f1", size = 1283704, upload-time = "2026-08-16T22:54:27.279Z" }, + { url = "https://files.pythonhosted.org/packages/46/2a/90cc8d7463929c04786f29600de45f3227c12fa9bed1d5b7ce319b05e1c9/hypothesis-6.165.10-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:10d9a650a4666b0914831f769703d36140ed8039fd19bf9b71f615b8541eccf2", size = 1325077, upload-time = "2026-08-16T22:55:16.561Z" }, + { url = "https://files.pythonhosted.org/packages/82/ac/bc16faba4b42883e3d290bfaceff51e258b63fbbdf789bf9fe88df1ce537/hypothesis-6.165.10-cp311-cp311-win_amd64.whl", hash = "sha256:5671d2b2bf83bd4b6f02e55b32d432506eff5358c82f39b460a849ce19a2666e", size = 674920, upload-time = "2026-08-16T22:55:42.613Z" }, + { url = "https://files.pythonhosted.org/packages/e9/45/cde4f78afe2b9e29caecf38319eedc1deb76aebcacbdd128e03cbb2511c3/hypothesis-6.165.10-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:637445c1593a2a9d1024fda50082f07bb56baedda78d90a25f64b8111727ef94", size = 784835, upload-time = "2026-08-16T22:54:45.429Z" }, + { url = "https://files.pythonhosted.org/packages/7f/81/847f30b81cbfd07607296b3ce43067cf4f80799bd9244167f587de9c8081/hypothesis-6.165.10-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:713f4ce4e82c26b53031f139de959bc9e8b54d3995aa824b89bbdf8229df2a45", size = 776419, upload-time = "2026-08-16T22:55:33.633Z" }, + { url = "https://files.pythonhosted.org/packages/04/66/4c71c5be7a49d84b8c3a9278c1807c4c81181ab5474beb27df9d4c40dc0e/hypothesis-6.165.10-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f9ff356e97e3ab09db07c8b675efa67340103874a0bae7465acb83dad7a35f7f", size = 1106830, upload-time = "2026-08-16T22:55:10.389Z" }, + { url = "https://files.pythonhosted.org/packages/e3/c4/e2cbd2810e79f7a452a8ea9f6c6438ee718ce938d8cc12252cf0b36a81d3/hypothesis-6.165.10-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a380bc99aa3b035e6a95a2201bf792d4082a04ca75babcc21849c2d0914bb28", size = 1156952, upload-time = "2026-08-16T22:55:53.35Z" }, + { url = "https://files.pythonhosted.org/packages/a8/8b/794ced36864825492ac3712d5acab5a257b4601e6a9dc2ccdd3937198f87/hypothesis-6.165.10-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e9acb2c4d9cb532c3fedea74159f7b923c8c036328c9239b4049e7aa073bdd81", size = 1280780, upload-time = "2026-08-16T22:54:34.983Z" }, + { url = "https://files.pythonhosted.org/packages/5d/2d/550525442cdbcc2daf1f9bdd8ba35bcbde63db7c7a22f2ef137fbb49df2f/hypothesis-6.165.10-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8660572b2d424bf5369ea8990985225f70bd1615b76ecd9c25588a3b9307009f", size = 1324130, upload-time = "2026-08-16T22:55:48.659Z" }, + { url = "https://files.pythonhosted.org/packages/74/59/6caf69dd5fe03499ada94c9cec016bffcc164511c6b93fe680f01209b9ff/hypothesis-6.165.10-cp312-cp312-win_amd64.whl", hash = "sha256:3376f2594763aef14faa519b0fb27cae7ce9eeaab4c69efa07777499110306c9", size = 672337, upload-time = "2026-08-16T22:54:49.11Z" }, + { url = "https://files.pythonhosted.org/packages/b1/fb/c82c5bd92864ffcf319772fedc8c9bf2dbe4ca14baa0fee6e49e67b5ba1c/hypothesis-6.165.10-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:9d77c3be7b429875036ad0f0597c6e5cc6bb17894a4da005e3807de64d2673ad", size = 784726, upload-time = "2026-08-16T22:54:32.371Z" }, + { url = "https://files.pythonhosted.org/packages/0e/b9/3d7acd08506da85557e65147b7f3fca8c47684e33be90bee0acb523920db/hypothesis-6.165.10-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:490c56b830772b0eca3b4b2cecb3741a1ed26b1d7206a279e1525dbf0aa95ee4", size = 776375, upload-time = "2026-08-16T22:55:13.303Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/922e8b3f9a706dd89d440b9545d2c6231c65e74da1c1fee3ff36c251b9c4/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed68e27b8a61e57a3ccdc7c5a14499e00b54dfe223087204d5d40b3b5ef58b6d", size = 1106763, upload-time = "2026-08-16T22:55:06.129Z" }, + { url = "https://files.pythonhosted.org/packages/01/39/f5b9a5d390d4edd1ad472334493ac442963ebeb4daaa74ff4bdac6ef292f/hypothesis-6.165.10-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6caadcd1afb62630ff5c5ff353626eaa616553a5971295ad6dc2b19ca8a39620", size = 1156778, upload-time = "2026-08-16T22:54:33.824Z" }, + { url = "https://files.pythonhosted.org/packages/b5/5f/5fbe1be4326337fd6acefe2d18ed44007ee1dc1f98fe5b3c0eb22942364d/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:d9145fe43ebb22e66672967c3fab411793b226ed776e4fe282271bca6ad3c0bb", size = 1280756, upload-time = "2026-08-16T22:55:54.834Z" }, + { url = "https://files.pythonhosted.org/packages/25/c0/cf6f9e1ef632a1a75694eed0db3a02e6fc75c367a363e94acee52f043c64/hypothesis-6.165.10-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:79900a9920a0b1d3a626c03a90ac6bf7042e78d46906a565b86a0dbe926f1d96", size = 1323889, upload-time = "2026-08-16T22:55:56.567Z" }, + { url = "https://files.pythonhosted.org/packages/cc/cc/662b94880f260b0a88de1fdcf60fc9984f6e2a796da549542adc10a7bc83/hypothesis-6.165.10-cp313-cp313-win_amd64.whl", hash = "sha256:c01dd04044c472e47193b54f68e84e08d6ebf4f29551885aa959b015f7cd9747", size = 672346, upload-time = "2026-08-16T22:56:03.792Z" }, + { url = "https://files.pythonhosted.org/packages/3f/77/55e020c9c576532ff7d20bf8b1dfa052ecbd5ada1949b02f76c44c966f7e/hypothesis-6.165.10-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:9ccac776b2ca93b324806facd526ccb45da0fd035001c899a35b02c44431e209", size = 784833, upload-time = "2026-08-16T22:55:21.255Z" }, + { url = "https://files.pythonhosted.org/packages/4f/f2/01da2adf829cf549eaddcabb8e8072077fb3d26da4275f4c1e89b2c0af74/hypothesis-6.165.10-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e5f95f7b622e4171096d92175dda0a560f0955ade9b8a3a07bdcf151f7359611", size = 776545, upload-time = "2026-08-16T22:56:10.159Z" }, + { url = "https://files.pythonhosted.org/packages/cf/8e/58d4f842895220b793c53fc94a6489705b3665bb4d0ae4d338ce03fdf9fb/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f76d1562643693b8a40066f1f96af795b93fd9bcfc9690a1af2ff4c5867ee29e", size = 1107271, upload-time = "2026-08-16T22:54:50.266Z" }, + { url = "https://files.pythonhosted.org/packages/8f/b8/206468912d2153306bb8a41afdfc59e45b7a73a0495bbe4b9cb4f0e79c1d/hypothesis-6.165.10-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60cab3ab4ea468d31a33739ffd7e94ec3e37dea891d65a6582ecc8a477175191", size = 1156915, upload-time = "2026-08-16T22:54:25.89Z" }, + { url = "https://files.pythonhosted.org/packages/fb/d3/bf5a22929b70a4cfd3edf69c5642b029b27ddb5cfda48fa295d384b01abb/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:22cf19388f0ff6ced8eb3e49c903d14938e4ed909d93bf28383eef451511e424", size = 1281205, upload-time = "2026-08-16T22:54:44.083Z" }, + { url = "https://files.pythonhosted.org/packages/07/a2/d7b2ba444d36fc84d4779f4431e74dd9b023dc63bcf282199f6e48ad39f4/hypothesis-6.165.10-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:057d0232f1224dcd0b7698902551a4341a7399f90670b036db6c4376715fe889", size = 1324243, upload-time = "2026-08-16T22:55:41.123Z" }, + { url = "https://files.pythonhosted.org/packages/d1/95/afe6b531fd01928c6f63d394ee413fa2338d088b2b44efcc23596b54477e/hypothesis-6.165.10-cp314-cp314-pyemscripten_2026_0_wasm32.whl", hash = "sha256:ab0f2e9d7d7d4db257f7cf53de3706c2baf124269571f20ffc2bcd6781f03063", size = 616382, upload-time = "2026-08-16T22:55:18.449Z" }, + { url = "https://files.pythonhosted.org/packages/48/86/9b4fb75f520a028edec50ffc904a94d724180395d71feb6d7a0ce7bb6f00/hypothesis-6.165.10-cp314-cp314-win_amd64.whl", hash = "sha256:d1ea02fa8ab3d33eb1125eade81f7136341eb429152c6dbe2ae6f8bc33b3fbdd", size = 672145, upload-time = "2026-08-16T22:54:24.831Z" }, + { url = "https://files.pythonhosted.org/packages/f9/ba/f7bbaae0c789bab7ddb764d2056ee1a463cc95a8acbccc90d4184e48b242/hypothesis-6.165.10-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:ed1a5891e59472884a03cb9875483e8fc131c80a275c60967f8afc5458a0c8ff", size = 783287, upload-time = "2026-08-16T22:54:23.751Z" }, + { url = "https://files.pythonhosted.org/packages/3a/83/01ef80772b4abd335c49405576dc503cede94fb5da30ba2643a119013aea/hypothesis-6.165.10-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:09772e328a26e50486ac572be34f9887f9aa185efe7ebb16bde4e8f6038db1f4", size = 774991, upload-time = "2026-08-16T22:55:25.987Z" }, + { url = "https://files.pythonhosted.org/packages/a3/0b/f47506241f9d5a5a2efe4c65b6bf4830e9d9576e5d3779007a260699e608/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5cf3b612542ba174c9da4000b59a4f4c81e8d66f87509be85d3a1b71b5c36413", size = 1105499, upload-time = "2026-08-16T22:54:51.864Z" }, + { url = "https://files.pythonhosted.org/packages/84/fe/abb3909b7089835112fbe75bf00d817d733b3a8032759783db0a24ff1e56/hypothesis-6.165.10-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f69ec5be85ef508e206153bed8eafd03f7995dc464356c8bbb279a1e2b7d56f3", size = 1155685, upload-time = "2026-08-16T22:54:30.94Z" }, + { url = "https://files.pythonhosted.org/packages/73/2f/1964738921640184067121ae77414522fc3f0463fc26c6e25a4f3b8e42ca/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:dd207497bb985918409a1bb5db85d1875f74e1269487332113b73d1ee7c77647", size = 1279177, upload-time = "2026-08-16T22:54:40.179Z" }, + { url = "https://files.pythonhosted.org/packages/34/c5/312af8ae038d3af9cf3f7f1021c1abfe31c0d9035e4cf63519e0a7dc983e/hypothesis-6.165.10-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:00de0abdcf8c05c9d0eab735a3c49a276376b55151e6fcb903c2b39a90e5e5c3", size = 1322921, upload-time = "2026-08-16T22:54:42.7Z" }, + { url = "https://files.pythonhosted.org/packages/9e/e7/b0a2fde7570c090a1b914026266a421c751ef10138fffe37fe0ef9e675c0/hypothesis-6.165.10-cp314-cp314t-win_amd64.whl", hash = "sha256:cc2da5aa4edf14743fa9257e5ba3513963999f01211635702479d8e92b8207c8", size = 672147, upload-time = "2026-08-16T22:55:27.527Z" }, + { url = "https://files.pythonhosted.org/packages/47/fd/985aa564d6ffd06483d45a62b40d319df0a703cd8bc1d041de17d102fbaa/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_10_12_x86_64.whl", hash = "sha256:eeab73050ea58c13dd56e329f594c1dfe32ebd7bb169bbdf4f8ceefbc31ec6b5", size = 782882, upload-time = "2026-08-16T22:55:37.93Z" }, + { url = "https://files.pythonhosted.org/packages/f8/2c/6cc11151e450f72353a490940cd0db704680d07b78dc75dcc9f480e0d0e1/hypothesis-6.165.10-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:4c68e983d0007d014bb01ad4bcbba78bc432c73a1755ff36d5102ceefa18299a", size = 774584, upload-time = "2026-08-16T22:55:51.822Z" }, + { url = "https://files.pythonhosted.org/packages/10/39/ef26fa79c1738dfe9cdb1a3584fb6717d26429ca6c9d011cc4fdf08130c2/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7730d8197086f65d8969a991d6728a1d420a51b19fea06535c896cb43a1e05d0", size = 1104876, upload-time = "2026-08-16T22:54:58.937Z" }, + { url = "https://files.pythonhosted.org/packages/4e/f4/3fcc84e7637f42bf00d987093b9418083ac8db81b87392608a60f4b7c5fd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7a7980a898a3e6ebe4de1896a0507e3d519edb53fb9b4bda478c9fbeb6514558", size = 1133353, upload-time = "2026-08-16T22:54:28.635Z" }, + { url = "https://files.pythonhosted.org/packages/35/59/21c5c14179c38f8d0de3560e7f1825c083311b3013b63f817d7dc78dfcbd/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5820d009aedb7ae9cfd32f98b1ab0c0bbd6268379c4fab042218b6b655c63f8", size = 1132300, upload-time = "2026-08-16T22:56:08.539Z" }, + { url = "https://files.pythonhosted.org/packages/14/af/fbb56059961e416b2de7b9dc5352db2e8572bd5ea46892957e4c1e5548ab/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:37a7ac3d34220800e1107871cc391bca1b00439875925d7d821878b8b791f245", size = 1155175, upload-time = "2026-08-16T22:55:19.824Z" }, + { url = "https://files.pythonhosted.org/packages/0f/53/77fb0c2dad445858555429c4e06cf94a59ae8d2407dd6426b5af97c84828/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_31_riscv64.whl", hash = "sha256:dafa7c9dbe3d802f9bcdf261b29c8a70700fb22839947f06e471f62c46b6257f", size = 1109881, upload-time = "2026-08-16T22:55:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/a8/7b/d187f673ff30e6ada640953636f978ffe64a6332f756b64163c2277f8d0c/hypothesis-6.165.10-cp315-abi3.abi3t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:90915635b9648071129b0f72c0673cf8eac9eb84cfd445c5bedef30c714b1ec2", size = 1144963, upload-time = "2026-08-16T22:56:13.428Z" }, + { url = "https://files.pythonhosted.org/packages/e0/60/31d504e364134d60af23e5f6365db0da3cf4a51b3ed3d4836e5a2cff12cf/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:e1bbeb7c506b07ee0422cf9b2f7212fefa4240957f03526d38d27bc6743a0a48", size = 1278684, upload-time = "2026-08-16T22:55:22.971Z" }, + { url = "https://files.pythonhosted.org/packages/ef/e6/89d26834a08c02f8da149e541dd40d7a96f68d9722f43146e69a77436ed7/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_armv7l.whl", hash = "sha256:2b36aaffc88625a44f91074c5bbedfdefb9b376c38d1b3c342edcd2e4c8ed16c", size = 1407202, upload-time = "2026-08-16T22:55:14.949Z" }, + { url = "https://files.pythonhosted.org/packages/dc/61/20d1e72246867ea195440092e8bb422c7ddc2f271b87b5b65679d5532719/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_ppc64le.whl", hash = "sha256:18a3ea838ddea183388f8788750afa8494d79abb5358823be9782585f34445d3", size = 1261395, upload-time = "2026-08-16T22:56:05.448Z" }, + { url = "https://files.pythonhosted.org/packages/2a/9b/ebab6c3c2b90a16abb4119198178652d12aff83cc8ec2cfde5276c69fb1e/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_riscv64.whl", hash = "sha256:2a2567b3a03a4a5a7c575c191cfcce321a967df3727803817e75bffbbeaecabe", size = 1279213, upload-time = "2026-08-16T22:55:35.066Z" }, + { url = "https://files.pythonhosted.org/packages/23/78/69b219b524231d36eb20c792e1f01e7cb037e02bd0af1c29f77ed9a969c0/hypothesis-6.165.10-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:8001925fa3dde51cb574e4c9de4c7efe77c4e4d64bd2fd2ef61d5651f9d04f3d", size = 1322367, upload-time = "2026-08-16T22:54:21.279Z" }, + { url = "https://files.pythonhosted.org/packages/55/63/ad5cc153dcc72ae5e7905fb9b3585f3e48ce892a2d6366f90163e867a69d/hypothesis-6.165.10-cp315-abi3.abi3t-win32.whl", hash = "sha256:c6559380469295c4009215fe1cab561301591a3bee2e2fb3f4f96d2273a3affc", size = 666038, upload-time = "2026-08-16T22:56:11.797Z" }, + { url = "https://files.pythonhosted.org/packages/80/32/b62307b73fbc99f0a4381d6f9456df76fbcbb7a27ef7256e26f0376f48ea/hypothesis-6.165.10-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:30797f20ca45e57f526d2df872f63ba453cb4e1091ad542184a7a951af8da79d", size = 671941, upload-time = "2026-08-16T22:55:00.235Z" }, + { url = "https://files.pythonhosted.org/packages/c2/dd/e0f98add0548ef73ea7afac45da1fb8efc854d7f9931db568754d0f963f3/hypothesis-6.165.10-cp315-abi3.abi3t-win_arm64.whl", hash = "sha256:c53e9b1c36350df9965ec44d6c0d4e0bbbb38f720dd2b0e1256dc6524d411015", size = 669931, upload-time = "2026-08-16T22:55:50.205Z" }, + { url = "https://files.pythonhosted.org/packages/0b/6a/880d6eeed5c451fb40a66733dadec4a5d498628a4a7f6a8a5f633f4c6dcb/hypothesis-6.165.10-pp311-pypy311_pp73-macosx_10_12_x86_64.whl", hash = "sha256:34ee6402df6f31274d89119f1561b5f7489c97866afc5b7a3ed3a13d7e762802", size = 784644, upload-time = "2026-08-16T22:54:20.127Z" }, + { url = "https://files.pythonhosted.org/packages/27/e0/9e942bd3c3cf5ea0d5c0fd0905893bbfb6cefb7284c70fcc8033f8fdec38/hypothesis-6.165.10-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:277f41801e88dad2eba082f91a75632b7584ff64044ba2cf9dadf511b0d19cd0", size = 780515, upload-time = "2026-08-16T22:55:04.676Z" }, + { url = "https://files.pythonhosted.org/packages/19/32/f11a618415dc5fa9cdde41fea56c489f0814759527ae1ecd11a75a4558b9/hypothesis-6.165.10-pp311-pypy311_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:72df95fb1db41755b155c5f02106e0036a339250555c8d351d488704fd112cf9", size = 1109374, upload-time = "2026-08-16T22:56:00.241Z" }, + { url = "https://files.pythonhosted.org/packages/5e/6f/db49b719842297c2b71e0d81e5b8967d31215fb7389421abcb465ce7ed3f/hypothesis-6.165.10-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6e20a02775eb3cf0ffb4f0219b6d7c1f240336663d4e5d7028675ec247c790c4", size = 1159092, upload-time = "2026-08-16T22:55:58.57Z" }, + { url = "https://files.pythonhosted.org/packages/b2/2a/bf0bae84ba1cb3923d295973f1fe38ee867eaf90119e0d559116083be300/hypothesis-6.165.10-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:1ec53f08732e3cfd0342cbbd75dbd1b193c8f19390660466e536a748bb81f757", size = 676045, upload-time = "2026-08-16T22:55:45.514Z" }, +] + [[package]] name = "idna" version = "3.15" @@ -4430,6 +4522,7 @@ dev = [ { name = "diff-cover" }, { name = "fakeredis" }, { name = "fastapi-offline" }, + { name = "hypothesis" }, { name = "keyring" }, { name = "langfuse" }, { name = "openapi-core" }, @@ -4449,6 +4542,7 @@ dev = [ { name = "pytest-rerunfailures" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, + { name = "reportlab" }, { name = "requests-mock" }, { name = "responses" }, { name = "respx" }, @@ -4615,6 +4709,7 @@ dev = [ { name = "diff-cover", specifier = "==9.7.2" }, { name = "fakeredis", specifier = "==2.34.1" }, { name = "fastapi-offline", specifier = "==1.7.6" }, + { name = "hypothesis", specifier = "==6.165.10" }, { name = "keyring", specifier = "==25.7.0" }, { name = "langfuse", specifier = "==2.59.7" }, { name = "openapi-core", specifier = "==0.22.0" }, @@ -4634,6 +4729,7 @@ dev = [ { name = "pytest-rerunfailures", specifier = "==15.1" }, { name = "pytest-timeout", specifier = "==2.4.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, + { name = "reportlab", specifier = "==5.0.1" }, { name = "requests-mock", specifier = "==1.12.1" }, { name = "responses", specifier = "==0.26.0" }, { name = "respx", specifier = "==0.22.0" }, @@ -8189,6 +8285,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/80/35b4c33c804a165a7f55289afda3ea9e3eb6d15800341a2d66455c0f1f30/regex-2026.5.9-cp314-cp314t-win_arm64.whl", hash = "sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa", size = 275713, upload-time = "2026-05-09T23:15:16.98Z" }, ] +[[package]] +name = "reportlab" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/51/dbe28534ae12c852f61be91f039f343305fd1f34f1c66b8de75afae7a525/reportlab-5.0.1.tar.gz", hash = "sha256:ebd13154be1c8515e665de70bd2d303ae9ddc3ef47e44afd5116441ca0283a26", size = 3945711, upload-time = "2026-08-20T13:48:16.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/cb/dacbc268cb68d0428ea2cbd85266195a9ab3e677449589ddae59bd7542ac/reportlab-5.0.1-py3-none-any.whl", hash = "sha256:1c36e6bb0e71780c72331eba60da7f602e8d4389a8723825af71342e49d791e8", size = 1957258, upload-time = "2026-08-20T13:48:14.026Z" }, +] + [[package]] name = "requests" version = "2.34.0"