mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-05 08:07:05 +00:00
test: add OCR python-to-rust test parity ledger (WIP) (#39434)
* test: add OCR python-to-rust test parity ledger * feat: add ledger loader for OCR test parity data * feat: add drift audit for OCR test parity ledger (WIP, untested) * fix: correct drift in OCR test-parity ledger Two entries referenced a typo'd Python test name, four duplicated entries already tracked under TestProxySecurityGuard, five real Python tests in test_rust_bridge.py were untracked, and three real Rust custom_logger tests were missing from rust_only_tests. Found by running validate_ledger.py's audit against the live repo. * test: add regression coverage for the OCR ledger and audit script Covers schema validation, AST/regex test enumeration, drift detection on both the Python and Rust sides, LedgerDriftError content, and a live-repo clean-audit guard against future drift. * test: simplify ledger test to one drift-guard assertion Replace the ledger-internals unit tests with a single test that runs the real audit against the live repo and asserts every OCR test is accounted for (mapped, unmapped-with-reason, or rust_only), printing the exact diff on failure. * chore: move OCR test-parity ledger to core/ocr validate_sub_methods/ mixes strategy-catalog metadata with the ledger. Ledger data belongs under a per-function core/<function>/ path instead. * fix: point LEDGER_PATH at the new core/ocr location
This commit is contained in:
ishaan-berri
GitHub
parent
7a81ae98e6
commit
a5639b8e2a
11 changed files with 581 additions and 2 deletions
|
|
@ -9,9 +9,11 @@ from .catalog import load_catalog
|
|||
from .models import HarnessCase, Strategy
|
||||
from .runner import run_pytest
|
||||
from .ui import make_dashboard
|
||||
from .strategies.unit_tests.mapping_validator import FunctionReport, build_function_report
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
COVERAGE_ROOT = REPO_ROOT / "target" / "rust-python-harness"
|
||||
SDK_FUNCTION_CHOICES = ("ocr", "messages", "responses", "count_tokens")
|
||||
|
||||
|
||||
def _parser() -> argparse.ArgumentParser:
|
||||
|
|
@ -40,9 +42,17 @@ def _parser() -> argparse.ArgumentParser:
|
|||
action="append",
|
||||
default=[],
|
||||
dest="sdk_functions",
|
||||
choices=("ocr", "messages", "responses", "count_tokens"),
|
||||
choices=SDK_FUNCTION_CHOICES,
|
||||
help="run only this SDK function",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--validate-ledger",
|
||||
action="store_true",
|
||||
help=(
|
||||
"report Python<->Rust test-parity ledger gaps and drift instead of "
|
||||
"running the dashboard; narrow with --function"
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--plain",
|
||||
action="store_true",
|
||||
|
|
@ -100,7 +110,7 @@ def _interactive_filters(strategies: Sequence[Strategy]) -> tuple[set[str], set[
|
|||
)
|
||||
sdk_functions = _pick_values(
|
||||
"SDK functions",
|
||||
[(name, name) for name in ("ocr", "messages", "responses", "count_tokens")],
|
||||
[(name, name) for name in SDK_FUNCTION_CHOICES],
|
||||
)
|
||||
return strategy_ids, sdk_functions
|
||||
|
||||
|
|
@ -131,6 +141,38 @@ def _print_catalog(strategies: Sequence[Strategy]) -> None:
|
|||
print(f" {case.sdk_function:12} {case.coverage.value:14} {selectors}")
|
||||
|
||||
|
||||
def _print_function_report(report: FunctionReport) -> None:
|
||||
print(f"\n{report.sdk_function}")
|
||||
if report.ledger is None or report.audit is None:
|
||||
print(" no ledger yet")
|
||||
return
|
||||
ledger, audit = report.ledger, report.audit
|
||||
print(
|
||||
f" {ledger.mapped_count}/{ledger.total_count} python tests mapped to rust "
|
||||
f"({ledger.percentage}%)"
|
||||
)
|
||||
print(f" {len(ledger.rust_only_tests)} rust-only tests with no python counterpart")
|
||||
if audit.is_clean:
|
||||
print(" ledger is in sync with the live test files")
|
||||
return
|
||||
for label, items in (
|
||||
("ledger references a python test that no longer exists", audit.missing_python_tests),
|
||||
("python test exists but is not tracked in the ledger", audit.stale_python_tests),
|
||||
("ledger references a rust test that no longer exists", audit.missing_rust_tests),
|
||||
("rust test exists but is not tracked in the ledger", audit.stale_rust_tests),
|
||||
):
|
||||
for item in items:
|
||||
print(f" {label}: {item}")
|
||||
|
||||
|
||||
def _validate_ledger(sdk_functions: set[str]) -> int:
|
||||
functions = sdk_functions or set(SDK_FUNCTION_CHOICES)
|
||||
reports = tuple(build_function_report(function) for function in sorted(functions))
|
||||
for report in reports:
|
||||
_print_function_report(report)
|
||||
return 0 if all(report.is_clean for report in reports) else 1
|
||||
|
||||
|
||||
def main(argv: Sequence[str] | None = None) -> int:
|
||||
args = _parser().parse_args(argv)
|
||||
if args.coverage and importlib.util.find_spec("pytest_cov") is None:
|
||||
|
|
@ -138,6 +180,8 @@ def main(argv: Sequence[str] | None = None) -> int:
|
|||
"--coverage requires the project's pytest-cov dependency; run with "
|
||||
"`poetry run python -m tests.rust-python-harness --coverage`"
|
||||
)
|
||||
if args.validate_ledger:
|
||||
return _validate_ledger(set(args.sdk_functions))
|
||||
strategies = load_catalog()
|
||||
if args.list:
|
||||
_print_catalog(strategies)
|
||||
|
|
|
|||
0
tests/rust-python-harness/shared/__init__.py
Normal file
0
tests/rust-python-harness/shared/__init__.py
Normal file
0
tests/rust-python-harness/shared/parity/__init__.py
Normal file
0
tests/rust-python-harness/shared/parity/__init__.py
Normal file
136
tests/rust-python-harness/shared/parity/ledger.py
Normal file
136
tests/rust-python-harness/shared/parity/ledger.py
Normal file
|
|
@ -0,0 +1,136 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class LedgerEntry:
|
||||
python_file: str
|
||||
python_test: str
|
||||
status: str
|
||||
rust_file: str
|
||||
rust_test: str
|
||||
justification: str
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class RustOnlyEntry:
|
||||
rust_file: str
|
||||
rust_test: str
|
||||
reason: str
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class TestLedger:
|
||||
sdk_function: str
|
||||
python_scope: tuple[str, ...]
|
||||
rust_scope: tuple[str, ...]
|
||||
entries: tuple[LedgerEntry, ...]
|
||||
rust_only_tests: tuple[RustOnlyEntry, ...]
|
||||
|
||||
@property
|
||||
def mapped_count(self) -> int:
|
||||
return sum(1 for entry in self.entries if entry.status == "mapped")
|
||||
|
||||
@property
|
||||
def total_count(self) -> int:
|
||||
return len(self.entries)
|
||||
|
||||
@property
|
||||
def percentage(self) -> float:
|
||||
if self.total_count == 0:
|
||||
return 0.0
|
||||
return round(100.0 * self.mapped_count / self.total_count, 1)
|
||||
|
||||
|
||||
def _require_string(value: Any, field: str, source: Path) -> str:
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ValueError(f"{source}: {field} must be a non-empty string")
|
||||
return value
|
||||
|
||||
|
||||
def _require_string_list(value: Any, field: str, source: Path) -> tuple[str, ...]:
|
||||
if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
|
||||
raise ValueError(f"{source}: {field} must be a list of non-empty strings")
|
||||
return tuple(value)
|
||||
|
||||
|
||||
def _load_entry(data: Any, index: int, source: Path) -> LedgerEntry:
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{source}: entries[{index}] must be an object")
|
||||
python_file = _require_string(data.get("python_file"), f"entries[{index}].python_file", source)
|
||||
python_test = _require_string(data.get("python_test"), f"entries[{index}].python_test", source)
|
||||
status = data.get("status")
|
||||
if status not in ("mapped", "unmapped"):
|
||||
raise ValueError(f"{source}: entries[{index}].status must be 'mapped' or 'unmapped'")
|
||||
|
||||
if status == "mapped":
|
||||
rust_file = _require_string(data.get("rust_file"), f"entries[{index}].rust_file", source)
|
||||
rust_test = _require_string(data.get("rust_test"), f"entries[{index}].rust_test", source)
|
||||
justification = _require_string(
|
||||
data.get("justification"), f"entries[{index}].justification", source
|
||||
)
|
||||
return LedgerEntry(
|
||||
python_file=python_file,
|
||||
python_test=python_test,
|
||||
status=status,
|
||||
rust_file=rust_file,
|
||||
rust_test=rust_test,
|
||||
justification=justification,
|
||||
reason="",
|
||||
)
|
||||
|
||||
reason = _require_string(data.get("reason"), f"entries[{index}].reason", source)
|
||||
return LedgerEntry(
|
||||
python_file=python_file,
|
||||
python_test=python_test,
|
||||
status=status,
|
||||
rust_file="",
|
||||
rust_test="",
|
||||
justification="",
|
||||
reason=reason,
|
||||
)
|
||||
|
||||
|
||||
def _load_rust_only_entry(data: Any, index: int, source: Path) -> RustOnlyEntry:
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError(f"{source}: rust_only_tests[{index}] must be an object")
|
||||
return RustOnlyEntry(
|
||||
rust_file=_require_string(data.get("rust_file"), f"rust_only_tests[{index}].rust_file", source),
|
||||
rust_test=_require_string(data.get("rust_test"), f"rust_only_tests[{index}].rust_test", source),
|
||||
reason=_require_string(data.get("reason"), f"rust_only_tests[{index}].reason", source),
|
||||
)
|
||||
|
||||
|
||||
def load_ledger(path: Path) -> TestLedger:
|
||||
with path.open(encoding="utf-8") as stream:
|
||||
data = json.load(stream)
|
||||
|
||||
sdk_function = _require_string(data.get("sdk_function"), "sdk_function", path)
|
||||
python_scope = _require_string_list(data.get("python_scope"), "python_scope", path)
|
||||
rust_scope = _require_string_list(data.get("rust_scope"), "rust_scope", path)
|
||||
|
||||
entries_data = data.get("entries")
|
||||
if not isinstance(entries_data, list):
|
||||
raise ValueError(f"{path}: entries must be a list")
|
||||
entries = tuple(
|
||||
_load_entry(entry, index, path) for index, entry in enumerate(entries_data)
|
||||
)
|
||||
|
||||
rust_only_data = data.get("rust_only_tests")
|
||||
if not isinstance(rust_only_data, list):
|
||||
raise ValueError(f"{path}: rust_only_tests must be a list")
|
||||
rust_only_tests = tuple(
|
||||
_load_rust_only_entry(entry, index, path) for index, entry in enumerate(rust_only_data)
|
||||
)
|
||||
|
||||
return TestLedger(
|
||||
sdk_function=sdk_function,
|
||||
python_scope=python_scope,
|
||||
rust_scope=rust_scope,
|
||||
entries=entries,
|
||||
rust_only_tests=rust_only_tests,
|
||||
)
|
||||
0
tests/rust-python-harness/strategies/__init__.py
Normal file
0
tests/rust-python-harness/strategies/__init__.py
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
{
|
||||
"sdk_function": "ocr",
|
||||
"python_scope": [
|
||||
"tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py",
|
||||
"tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py",
|
||||
"tests/test_litellm/ocr/test_rust_bridge.py",
|
||||
"tests/test_litellm/ocr/test_ocr_file_input.py",
|
||||
"tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py",
|
||||
"tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py",
|
||||
"tests/test_litellm/ocr/test_ocr_native_format.py",
|
||||
"tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py",
|
||||
"tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py"
|
||||
],
|
||||
"rust_scope": [
|
||||
"litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs",
|
||||
"litellm-rust/crates/ai-gateway/src/ocr/tests.rs",
|
||||
"litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs",
|
||||
"litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs",
|
||||
"litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs",
|
||||
"litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs"
|
||||
],
|
||||
"entries": [
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_encode_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id URL percent-encoding has no Rust test; Rust only tests pages/features query building"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_should_reject_dot_segment_azure_document_intelligence_model_id", "status": "unmapped", "reason": "model-id dot-segment validation has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "both assert page markdown, dimension (inch-to-pixel) normalization, and usage_info.pages_processed from the same Azure succeeded response shape"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_preserves_azure_native_fields", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_response_normalizes_pages", "justification": "async twin of the sync case above, same underlying transform is exercised on the Rust side"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_tolerates_missing_native_fields", "status": "unmapped", "reason": "tables/keyValuePairs absence tolerance is not asserted by the Rust response test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_non_succeeded_status_raises", "status": "unmapped", "reason": "no Rust test asserts on a non-succeeded Azure DI status"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_supported_ocr_params_includes_features", "status": "unmapped", "reason": "supported-params list content has no Rust equivalent for Azure"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_async_transform_ocr_response_native_format_carries_raw_operation", "status": "unmapped", "reason": "native req_format raw-operation passthrough is not tested in Rust"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_transform_ocr_response_default_format_omits_raw_operation", "status": "unmapped", "reason": "req_format gating of raw-operation output has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_passes_through_req_format", "status": "unmapped", "reason": "req_format passthrough in map_ocr_params has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_rejects_unknown_req_format_as_bad_request", "status": "unmapped", "reason": "req_format validation error path has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_omits_req_format_query_param", "status": "unmapped", "reason": "no Rust test asserts req_format is excluded from the built URL"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_features", "status": "unmapped", "reason": "features-string normalization in map_ocr_params has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_empty_features_list_omitted", "status": "unmapped", "reason": "empty-features omission has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_map_ocr_params_invalid_features_raises", "status": "unmapped", "reason": "features validation error path has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_appends_features_query", "status": "unmapped", "reason": "features query-param construction has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_get_complete_url_combines_pages_and_features", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_url_normalizes_zero_based_pages", "justification": "both assert 0-based, duplicate page indices are deduped, sorted, and rewritten 1-based into the request URL"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_uses_subscription_key", "status": "unmapped", "reason": "Python-side header derivation from litellm_params; Rust's poll test only checks the header is present, not how it was resolved"},
|
||||
{"python_file": "tests/test_litellm/llms/azure_ai/test_azure_document_intelligence_ocr_transformation.py", "python_test": "test_validate_environment_falls_back_to_entra_token", "status": "unmapped", "reason": "Entra bearer-token fallback logic has no Rust test"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_doc_intelligence_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_matches_documentintelligence_and_is_case_insensitive", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestIsAzureDocumentIntelligenceModel::test_does_not_match_mistral_route", "status": "unmapped", "reason": "model-route string matching is Python-only dispatch logic"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_does_not_hijack_doc_intelligence", "status": "unmapped", "reason": "api_base resolution from the secret manager runs before the Rust bridge is called, no Rust test exists for it"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_explicit_api_base_is_honoured_for_doc_intelligence", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_azure_document_intelligence_api_base.py", "python_test": "TestDocIntelligenceApiBaseResolution::test_generic_azure_ai_base_still_applies_to_mistral_ocr", "status": "unmapped", "reason": "api_base precedence resolution is Python-only"},
|
||||
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_use_litellm_rust_toggles_flag", "status": "unmapped", "reason": "bridge-plumbing: Python-side feature-flag toggle, no Rust equivalent"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_env_var_enables_rust_ocr", "status": "unmapped", "reason": "bridge-plumbing: Python-side env-var flag gating"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook, not provider behavior"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_returns_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: native-extension import/loader fallback"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_loader_caches_absent_extension", "status": "unmapped", "reason": "bridge-plumbing: loader caching behavior"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_native_bridge_available_reflects_loader", "status": "unmapped", "reason": "bridge-plumbing: loader availability check"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_aocr_returns_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: dependency-injection test hook"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_toggle_without_ocr_arg_preserves_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl state retention regression"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_explicit_ocr_none_clears_injected_impl", "status": "unmapped", "reason": "bridge-plumbing: injected-impl clearing behavior"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_none_when_extension_absent", "status": "unmapped", "reason": "bridge-plumbing: degrade path when the native extension is missing"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_load_rust_ocr_uses_compiled_extension", "status": "unmapped", "reason": "bridge-plumbing: native module resolution"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_timeout_to_seconds_handles_float_timeout_and_none", "status": "unmapped", "reason": "bridge-plumbing: Python-side timeout normalization helper"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: wrapper argument forwarding, asserted against a fake bridge not the real Rust code"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: async wrapper argument forwarding"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prepares_request_and_wraps_response", "status": "unmapped", "reason": "bridge-plumbing: request preparation and response wrapping in Python"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_resolves_key_via_secret_manager_when_missing", "status": "unmapped", "reason": "secret-manager: API key resolution happens in Python before the bridge is invoked"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_prefers_explicit_key_over_resolver", "status": "unmapped", "reason": "secret-manager: key precedence resolution"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_uses_provider_api_key_env_var", "status": "unmapped", "reason": "secret-manager: provider-specific env var name resolution"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_forwards_vertex_routing_metadata", "status": "unmapped", "reason": "secret-manager: vertex routing metadata merge happens in Python"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_vertex_routing_metadata_from_secret_manager", "status": "unmapped", "reason": "secret-manager: vertex project/location resolution"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_azure_ai_api_base_from_secret_manager", "status": "unmapped", "reason": "secret-manager: azure_ai api_base resolution"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_prepare_rust_ocr_call_resolves_document_intelligence_endpoint", "status": "unmapped", "reason": "secret-manager: doc-intelligence endpoint resolution"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_run_rust_ocr_runs_pre_call_logging", "status": "unmapped", "reason": "bridge-plumbing: Python logging-object pre_call invocation"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: routing to a fake bridge, not the real Rust transform"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_routes_azure_ai_to_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: provider-prefix stripping before routing"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_rust_path_converts_file_document_before_bridge", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion happens in Python before the bridge call"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: Python exception-type mapping on bridge failure"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_routes_to_async_rust_when_enabled", "status": "unmapped", "reason": "bridge-plumbing: async routing to a fake bridge"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_aocr_exception_type_uses_resolved_provider_context", "status": "unmapped", "reason": "bridge-plumbing: async exception-type mapping on bridge failure"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_does_not_route_to_rust_when_disabled", "status": "unmapped", "reason": "bridge-plumbing: Python control flow for the toggle-disabled branch, no Rust-owned behavior runs"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_falls_back_to_python_when_bridge_unavailable", "status": "unmapped", "reason": "bridge-plumbing: Python-only fallback when the compiled Rust extension is absent, Rust cannot test its own absence"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_forwards_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site forwards a timeout kwarg, Rust receives an already-constructed request"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_passes_default_request_timeout_to_rust", "status": "unmapped", "reason": "bridge-plumbing: asserts the Python call site supplies a default timeout kwarg, no Rust equivalent"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_rust_bridge.py", "python_test": "test_ocr_provider_configs_expose_api_key_env_vars", "status": "unmapped", "reason": "asserts per-provider get_api_key_env_var() strings; the closest Rust test (ocr_dispatch_supports_migrated_providers) asserts provider dispatch/param resolution instead, not API key env var names"},
|
||||
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_pdf_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection is Python-only preprocessing before the bridge call"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_png_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_jpeg_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_gif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_webp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tiff_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_tif_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_detect_bmp_mime_type", "status": "unmapped", "reason": "file-normalization: MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_be_case_insensitive", "status": "unmapped", "reason": "file-normalization: MIME detection case handling"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestGetMimeType::test_should_fallback_for_unknown_extension", "status": "unmapped", "reason": "file-normalization: MIME detection fallback"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pdf_pathlib_path_to_document_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion happens in Python"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_image_pathlib_path_to_image_url", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_reject_bare_str_path", "status": "unmapped", "reason": "file-normalization: arbitrary-file-read guard on bare str paths"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_pathlib_path", "status": "unmapped", "reason": "file-normalization: local-path-to-data-URI conversion"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes", "status": "unmapped", "reason": "file-normalization: raw-bytes-to-data-URI conversion"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_explicit_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_raw_bytes_with_image_mime_type", "status": "unmapped", "reason": "file-normalization: explicit MIME override on raw bytes"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object", "status": "unmapped", "reason": "file-normalization: file-like-object conversion"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_convert_file_like_object_with_name", "status": "unmapped", "reason": "file-normalization: file-like-object name-based MIME detection"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_missing_file_field", "status": "unmapped", "reason": "file-normalization: missing-field validation"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_nonexistent_pathlib_path", "status": "unmapped", "reason": "file-normalization: missing-file validation"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_empty_file", "status": "unmapped", "reason": "file-normalization: empty-file validation"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_unsupported_type", "status": "unmapped", "reason": "file-normalization: unsupported input type validation"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_raise_error_for_invalid_mime_type", "status": "unmapped", "reason": "file-normalization: MIME-type injection validation"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestConvertFileDocumentToUrlDocument::test_should_override_mime_type_for_pathlib_path", "status": "unmapped", "reason": "file-normalization: explicit MIME override precedence"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_document_url_for_pdf", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_png", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_build_image_url_for_jpeg", "status": "unmapped", "reason": "file-normalization: multipart upload conversion"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_octet_stream", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_detect_mime_from_filename_when_content_type_is_none", "status": "unmapped", "reason": "file-normalization: filename-based MIME fallback"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_fallback_to_octet_stream_for_unknown", "status": "unmapped", "reason": "file-normalization: default MIME fallback"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_preserve_base64_content_correctly", "status": "unmapped", "reason": "file-normalization: binary round-trip through base64"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_from_content_type", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestBuildDocumentFromUpload::test_should_strip_mime_parameters_with_multiple_params", "status": "unmapped", "reason": "file-normalization: content-type parameter stripping"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_reject_file_type_document_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body file-type guard, a different mechanism than Rust's URL-fetch SSRF guard"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_accept_document_url_type_in_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_raise_on_invalid_json_body", "status": "unmapped", "reason": "proxy-layer JSON-body parsing error path"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_file_input.py", "python_test": "TestProxySecurityGuard::test_should_ignore_document_form_field_injection", "status": "unmapped", "reason": "proxy-layer multipart form-field injection guard, a different mechanism than Rust's URL-fetch SSRF guard"},
|
||||
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_header_in_supported_params", "status": "unmapped", "reason": "Rust's fixed-list test checks the full list as one assertion, not this individual param"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_extract_footer_in_supported_params", "status": "unmapped", "reason": "Rust's fixed-list test checks the full list as one assertion, not this individual param"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestGetSupportedOcrParams::test_existing_params_still_present", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "supported_params_match_python_mistral_ocr_config", "justification": "both assert the full supported_ocr_params list matches the same fixed set of param names"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_passed_through", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert extract_header survives map_ocr_params filtering unchanged"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_footer_passed_through", "status": "unmapped", "reason": "Rust's map_ocr_params test does not assert on extract_footer specifically"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_extract_header_and_footer_together", "status": "unmapped", "reason": "combined extract_header+extract_footer passthrough is not asserted together in Rust"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestMapOcrParams::test_unknown_param_is_dropped", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "map_ocr_params_drops_unknown_params", "justification": "both assert an unrecognized param key is dropped while a known one is kept"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewSupportedParams::test_new_param_in_supported_list", "status": "unmapped", "reason": "OCR4-specific new params (table_format etc) are not individually verified against the Rust fixed-list test"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestNewParamsMapOcr::test_new_param_passed_through", "status": "unmapped", "reason": "OCR4-specific new params are not individually asserted in the Rust map_ocr_params test"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_param_included_in_request_body", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_builds_mistral_body", "justification": "both assert an optional param value ends up in the built request body alongside model/document"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrRequest::test_multiple_new_params_together", "status": "mapped", "rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_builds_mistral_body", "justification": "both assert multiple optional params (table_format/include_image_base64) land correctly in the same request body"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_blocks_and_confidence_scores_preserved", "status": "unmapped", "reason": "OCR4 blocks/confidence_scores fields are not asserted by the Rust response test"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_transformation.py", "python_test": "TestTransformOcrResponseOcr4Fields::test_ocr4_fields_survive_model_dump", "status": "unmapped", "reason": "OCR4 tables/hyperlinks/header/footer fields are not asserted by the Rust response test"},
|
||||
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_model_info_ocr4_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr4_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_pricing_entry", "status": "unmapped", "reason": "cost-calc: cost-map JSON entry validation is Python-only"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_model_info_price", "status": "unmapped", "reason": "cost-calc: pricing/model-info lookup is Python-only"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_cost_scales_with_pages", "status": "unmapped", "reason": "cost-calc: per-page pricing math is Python-only"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_ocr_and_annotation_pages_at_their_own_rates", "status": "unmapped", "reason": "cost-calc: mixed-rate billing math is Python-only"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_only_response", "status": "unmapped", "reason": "cost-calc: annotation-only billing math is Python-only"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_ocr3_bills_annotation_pages_when_pages_processed_missing", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"},
|
||||
{"python_file": "tests/test_litellm/llms/mistral/ocr/test_mistral_ocr_cost.py", "python_test": "test_azure_doc_ai_annotation_pages_fall_back_to_ocr_rate", "status": "unmapped", "reason": "cost-calc: fallback billing math is Python-only"},
|
||||
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_serves_default_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_rust_ocr_skipped_for_native_format", "status": "unmapped", "reason": "request-format gating decision is made in Python before the Rust bridge is ever invoked"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_native_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "provider-support validation for req_format happens in Python"},
|
||||
{"python_file": "tests/test_litellm/ocr/test_ocr_native_format.py", "python_test": "test_unknown_format_rejected_for_provider_without_support_as_bad_request", "status": "unmapped", "reason": "req_format validation error path is Python-only"},
|
||||
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_ocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestHandlerDiscovery::test_handler_discovered_for_aocr", "status": "unmapped", "reason": "guardrail-translation handler discovery is a Python proxy-layer concern"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_document_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_image_url", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_no_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_process_invalid_document", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestInputProcessing::test_input_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_single_page", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_multiple_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_empty_pages", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_page_with_empty_markdown", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_process_preserves_page_metadata", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestOutputProcessing::test_output_blocking_guardrail", "status": "unmapped", "reason": "scoped to the Python translation handler, not the Rust gateway's guardrail hook lifecycle"},
|
||||
{"python_file": "tests/test_litellm/llms/ocr/guardrail_translation/test_ocr_guardrail_handler.py", "python_test": "TestPIIMaskingScenario::test_pii_masking_in_ocr_pages", "status": "unmapped", "reason": "PII redaction in the translation handler has no Rust equivalent"},
|
||||
|
||||
{"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_read_req_format_from_header", "status": "unmapped", "reason": "proxy-layer header parsing has no Rust equivalent"},
|
||||
{"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_prefer_body_req_format_over_header", "status": "unmapped", "reason": "proxy-layer body-vs-header precedence has no Rust equivalent"},
|
||||
{"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_omit_req_format_when_header_absent", "status": "unmapped", "reason": "proxy-layer parsing has no Rust equivalent"},
|
||||
{"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_reject_unknown_req_format", "status": "unmapped", "reason": "proxy-layer validation has no Rust equivalent"},
|
||||
{"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_native_payload_with_litellm_response_headers", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"},
|
||||
{"python_file": "tests/test_litellm/proxy/ocr_endpoints/test_endpoints.py", "python_test": "test_should_return_normalized_response_when_no_native_payload", "status": "unmapped", "reason": "proxy-layer response construction has no Rust equivalent"}
|
||||
],
|
||||
"rust_only_tests": [
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_failure_payload_for_non_ocr_call_type", "reason": "exercises the non-OCR (acompletion) call-type branch of the logger; the OCR branch is covered separately by rust_custom_logger_reads_success_payload_for_ocr"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "no_callback_fast_path_dispatches_nothing", "reason": "Rust-only fast-path optimization test for when zero callbacks are registered; Python has no equivalent no-op dispatch path"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "with_standard_logging_payload_keeps_top_level_fields_in_sync", "reason": "Rust-internal builder-method invariant, Python has no equivalent internal builder"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "blocks_private_and_metadata_ips", "reason": "SSRF IP-blocking helper has no Python unit test; Python relies on the proxy-layer JSON/form guards instead"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_rejects_loopback_fetch", "reason": "URL-fetch SSRF protection is Rust-gateway-only"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/common_utils.rs", "rust_test": "convert_document_url_leaves_data_uri_untouched", "reason": "URL-fetch SSRF protection is Rust-gateway-only"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_passes_short_strings_through", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_caps_long_payloads", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "truncate_error_body_does_not_split_multibyte_chars", "reason": "Rust-gateway error-body truncation helper has no Python counterpart"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_dispatch_supports_migrated_providers", "reason": "Rust-internal provider-config dispatch table has no equivalent Python unit test"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_accepts_string_values", "reason": "Rust-gateway header-coercion helper has no Python counterpart"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "auth_header_detection_is_case_insensitive", "reason": "Rust-gateway header-detection helper has no Python counterpart"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_pre_during_and_success_hooks", "reason": "full gateway-level guardrail-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_runs_failure_hook_on_provider_error", "reason": "full gateway-level failure-hook-plus-HTTP-lifecycle test with no Python equivalent at this integration scope"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_lifecycle_pre_call_block_skips_provider_socket", "reason": "full gateway-level pre-call-block-plus-socket-skip test with no Python equivalent at this integration scope"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "ocr_does_not_duplicate_authorization_header_when_header_is_supplied", "reason": "outgoing HTTP header dedup at the Rust gateway has no Python counterpart"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "document_intelligence_poll_uses_resolved_subscription_key", "reason": "full Azure DI poll-loop integration test with no Python equivalent at this scope"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/ocr/tests.rs", "rust_test": "string_headers_rejects_non_string_values", "reason": "Rust-gateway header-coercion error path has no Python counterpart"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "azure_ai_reuses_mistral_body_transform", "reason": "Rust-internal delegation-to-Mistral-transform implementation detail, no Python test asserts this delegation"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs", "rust_test": "document_intelligence_request_uses_base64_source_for_data_uri", "reason": "no Python test asserts on the base64Source request body shape"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_url_uses_project_location_and_model", "reason": "vertex OCR support has no Python unit test coverage yet"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_mistral_reuses_mistral_body_transform", "reason": "vertex OCR support has no Python unit test coverage yet"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_request_uses_ocr_endpoint_shape", "reason": "vertex OCR support has no Python unit test coverage yet"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/vertex_ai/ocr/transformation.rs", "rust_test": "vertex_deepseek_response_wraps_markdown_content", "reason": "vertex OCR support has no Python unit test coverage yet"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_request_rejects_non_object_document", "reason": "non-object document rejection has no dedicated Python unit test"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "transform_ocr_response_normalizes_mistral_json", "reason": "Python's response tests target OCR4-specific fields only, none asserts the same base normalization this Rust test checks"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "complete_url_defaults_and_dedupes_v1", "reason": "URL-building/defaulting for Mistral has no Python unit test"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_prefers_param_then_env", "reason": "API key resolution precedence at the Rust provider-config layer has no Python unit test"},
|
||||
{"rust_file": "litellm-rust/crates/core/src/providers/mistral/ocr/transformation.rs", "rust_test": "resolve_api_key_errors_when_absent", "reason": "API key resolution error path at the Rust provider-config layer has no Python unit test"},
|
||||
{"rust_file": "litellm-rust/crates/ai-gateway/src/integrations/custom_logger/mod.rs", "rust_test": "rust_custom_logger_reads_success_payload_for_ocr", "reason": "Rust-internal custom-logger dispatch for OCR payloads has no Python unit test at this layer"}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,101 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from ...shared.parity.ledger import TestLedger, load_ledger
|
||||
from .python_runner import enumerate_python_tests
|
||||
from .rust_runner import enumerate_rust_tests
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
LEDGER_ROOT = Path(__file__).parent / "ledgers"
|
||||
|
||||
|
||||
def ledger_path_for(sdk_function: str) -> Path:
|
||||
return LEDGER_ROOT / sdk_function / f"{sdk_function}_test_ledger.json"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class AuditReport:
|
||||
missing_python_tests: tuple[str, ...]
|
||||
stale_python_tests: tuple[str, ...]
|
||||
missing_rust_tests: tuple[str, ...]
|
||||
stale_rust_tests: tuple[str, ...]
|
||||
|
||||
@property
|
||||
def is_clean(self) -> bool:
|
||||
return not (
|
||||
self.missing_python_tests
|
||||
or self.stale_python_tests
|
||||
or self.missing_rust_tests
|
||||
or self.stale_rust_tests
|
||||
)
|
||||
|
||||
|
||||
def _ledger_python_tests_by_file(ledger: TestLedger) -> dict[str, set[str]]:
|
||||
grouping: dict[str, set[str]] = {path: set() for path in ledger.python_scope}
|
||||
for entry in ledger.entries:
|
||||
grouping.setdefault(entry.python_file, set()).add(entry.python_test)
|
||||
return grouping
|
||||
|
||||
|
||||
def _ledger_rust_tests_by_file(ledger: TestLedger) -> dict[str, set[str]]:
|
||||
grouping: dict[str, set[str]] = {path: set() for path in ledger.rust_scope}
|
||||
for entry in ledger.entries:
|
||||
if entry.status == "mapped":
|
||||
grouping.setdefault(entry.rust_file, set()).add(entry.rust_test)
|
||||
for rust_only in ledger.rust_only_tests:
|
||||
grouping.setdefault(rust_only.rust_file, set()).add(rust_only.rust_test)
|
||||
return grouping
|
||||
|
||||
|
||||
def audit_ledger(ledger: TestLedger, repo_root: Path = REPO_ROOT) -> AuditReport:
|
||||
missing_python: list[str] = []
|
||||
stale_python: list[str] = []
|
||||
for python_file, ledger_tests in _ledger_python_tests_by_file(ledger).items():
|
||||
actual_tests = enumerate_python_tests(repo_root, python_file)
|
||||
for missing in sorted(ledger_tests - actual_tests):
|
||||
missing_python.append(f"{python_file}:{missing}")
|
||||
for stale in sorted(actual_tests - ledger_tests):
|
||||
stale_python.append(f"{python_file}:{stale}")
|
||||
|
||||
missing_rust: list[str] = []
|
||||
stale_rust: list[str] = []
|
||||
for rust_file, ledger_tests in _ledger_rust_tests_by_file(ledger).items():
|
||||
actual_tests = enumerate_rust_tests(repo_root, rust_file)
|
||||
for missing in sorted(ledger_tests - actual_tests):
|
||||
missing_rust.append(f"{rust_file}:{missing}")
|
||||
for stale in sorted(actual_tests - ledger_tests):
|
||||
stale_rust.append(f"{rust_file}:{stale}")
|
||||
|
||||
return AuditReport(
|
||||
missing_python_tests=tuple(missing_python),
|
||||
stale_python_tests=tuple(stale_python),
|
||||
missing_rust_tests=tuple(missing_rust),
|
||||
stale_rust_tests=tuple(stale_rust),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class FunctionReport:
|
||||
sdk_function: str
|
||||
ledger: TestLedger | None
|
||||
audit: AuditReport | None
|
||||
|
||||
@property
|
||||
def has_ledger(self) -> bool:
|
||||
return self.ledger is not None
|
||||
|
||||
@property
|
||||
def is_clean(self) -> bool:
|
||||
return self.audit is None or self.audit.is_clean
|
||||
|
||||
|
||||
def build_function_report(sdk_function: str, repo_root: Path = REPO_ROOT) -> FunctionReport:
|
||||
path = ledger_path_for(sdk_function)
|
||||
if not path.exists():
|
||||
return FunctionReport(sdk_function=sdk_function, ledger=None, audit=None)
|
||||
ledger = load_ledger(path)
|
||||
return FunctionReport(
|
||||
sdk_function=sdk_function, ledger=ledger, audit=audit_ledger(ledger, repo_root)
|
||||
)
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def enumerate_python_tests(repo_root: Path, relative_path: str) -> frozenset[str]:
|
||||
source = (repo_root / relative_path).read_text(encoding="utf-8")
|
||||
tree = ast.parse(source, filename=relative_path)
|
||||
|
||||
module_level: list[str] = []
|
||||
for node in ast.iter_child_nodes(tree):
|
||||
if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name.startswith("test_"):
|
||||
module_level.append(node.name)
|
||||
elif isinstance(node, ast.ClassDef):
|
||||
for child in ast.iter_child_nodes(node):
|
||||
if isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef)) and child.name.startswith(
|
||||
"test_"
|
||||
):
|
||||
module_level.append(f"{node.name}::{child.name}")
|
||||
|
||||
return frozenset(module_level)
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
_RUST_TEST_PATTERN = re.compile(
|
||||
r"#\[(?:test|tokio::test)\][^\n]*\n(?:[^\n]*\n)*?\s*(?:async\s+)?fn\s+(\w+)\s*\("
|
||||
)
|
||||
|
||||
|
||||
def enumerate_rust_tests(repo_root: Path, relative_path: str) -> frozenset[str]:
|
||||
source = (repo_root / relative_path).read_text(encoding="utf-8")
|
||||
return frozenset(match.group(1) for match in _RUST_TEST_PATTERN.finditer(source))
|
||||
|
|
@ -8,14 +8,24 @@ import pytest
|
|||
|
||||
catalog = importlib.import_module("tests.rust-python-harness.catalog")
|
||||
cli = importlib.import_module("tests.rust-python-harness.cli")
|
||||
ledger_module = importlib.import_module("tests.rust-python-harness.shared.parity.ledger")
|
||||
mapping_validator = importlib.import_module(
|
||||
"tests.rust-python-harness.strategies.unit_tests.mapping_validator"
|
||||
)
|
||||
models = importlib.import_module("tests.rust-python-harness.models")
|
||||
runner = importlib.import_module("tests.rust-python-harness.runner")
|
||||
ui = importlib.import_module("tests.rust-python-harness.ui")
|
||||
|
||||
load_catalog = catalog.load_catalog
|
||||
load_ledger = ledger_module.load_ledger
|
||||
ledger_path_for = mapping_validator.ledger_path_for
|
||||
REPO_ROOT = mapping_validator.REPO_ROOT
|
||||
audit_ledger = mapping_validator.audit_ledger
|
||||
build_function_report = mapping_validator.build_function_report
|
||||
_pick_values = cli._pick_values
|
||||
_coverage_pytest_args = cli._coverage_pytest_args
|
||||
_select = cli._select
|
||||
_validate_ledger = cli._validate_ledger
|
||||
CaseResult = models.CaseResult
|
||||
Coverage = models.Coverage
|
||||
HarnessCase = models.HarnessCase
|
||||
|
|
@ -234,3 +244,48 @@ def test_should_report_confidence_for_each_sdk_section() -> None:
|
|||
assert scores["responses"].level.value == "MEDIUM"
|
||||
assert scores["count_tokens"].percentage == 0
|
||||
assert scores["count_tokens"].level.value == "LOW"
|
||||
|
||||
|
||||
|
||||
def test_should_report_no_ledger_for_a_function_without_one() -> None:
|
||||
report = build_function_report("messages", repo_root=REPO_ROOT)
|
||||
|
||||
assert report.has_ledger is False
|
||||
assert report.is_clean is True
|
||||
|
||||
|
||||
def test_should_report_ocr_ledger_stats_and_a_clean_audit() -> None:
|
||||
ledger = load_ledger(ledger_path_for("ocr"))
|
||||
|
||||
report = build_function_report("ocr", repo_root=REPO_ROOT)
|
||||
|
||||
assert report.has_ledger is True
|
||||
assert report.ledger.mapped_count == ledger.mapped_count
|
||||
assert report.ledger.total_count == ledger.total_count
|
||||
assert report.is_clean is True
|
||||
|
||||
|
||||
def test_should_scope_validate_ledger_to_the_requested_function(
|
||||
capsys: pytest.CaptureFixture[str],
|
||||
) -> None:
|
||||
exit_code = _validate_ledger({"messages"})
|
||||
|
||||
captured = capsys.readouterr()
|
||||
assert exit_code == 0
|
||||
assert "messages" in captured.out
|
||||
assert "no ledger yet" in captured.out
|
||||
assert "ocr" not in captured.out
|
||||
|
||||
|
||||
def test_should_have_every_python_and_rust_ocr_test_accounted_for_in_the_ledger() -> None:
|
||||
ledger = load_ledger(ledger_path_for("ocr"))
|
||||
|
||||
report = audit_ledger(ledger, repo_root=REPO_ROOT)
|
||||
|
||||
assert report.is_clean, (
|
||||
"\nOCR test-parity ledger is out of sync with the live test files.\n"
|
||||
f"Ledger references a Python test that no longer exists: {list(report.missing_python_tests)}\n"
|
||||
f"Python test exists but is not tracked in the ledger: {list(report.stale_python_tests)}\n"
|
||||
f"Ledger references a Rust test that no longer exists: {list(report.missing_rust_tests)}\n"
|
||||
f"Rust test exists but is not tracked in the ledger: {list(report.stale_rust_tests)}\n"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue