litellm/tests/rust-python-harness/shared/parity/ledger.py
ishaan-berri a5639b8e2a
test: add OCR python-to-rust test parity ledger (WIP) (#39434)
* test: add OCR python-to-rust test parity ledger

* feat: add ledger loader for OCR test parity data

* feat: add drift audit for OCR test parity ledger (WIP, untested)

* fix: correct drift in OCR test-parity ledger

Two entries referenced a typo'd Python test name, four duplicated entries already tracked under TestProxySecurityGuard, five real Python tests in test_rust_bridge.py were untracked, and three real Rust custom_logger tests were missing from rust_only_tests. Found by running validate_ledger.py's audit against the live repo.

* test: add regression coverage for the OCR ledger and audit script

Covers schema validation, AST/regex test enumeration, drift detection on both the Python and Rust sides, LedgerDriftError content, and a live-repo clean-audit guard against future drift.

* test: simplify ledger test to one drift-guard assertion

Replace the ledger-internals unit tests with a single test that runs the real audit against the live repo and asserts every OCR test is accounted for (mapped, unmapped-with-reason, or rust_only), printing the exact diff on failure.

* chore: move OCR test-parity ledger to core/ocr

validate_sub_methods/ mixes strategy-catalog metadata with the ledger. Ledger data belongs under a per-function core/<function>/ path instead.

* fix: point LEDGER_PATH at the new core/ocr location
2026-09-02 19:22:52 -07:00

136 lines
4.6 KiB
Python

from __future__ import annotations
import json
from dataclasses import dataclass
from pathlib import Path
from typing import Any
@dataclass(frozen=True, slots=True)
class LedgerEntry:
python_file: str
python_test: str
status: str
rust_file: str
rust_test: str
justification: str
reason: str
@dataclass(frozen=True, slots=True)
class RustOnlyEntry:
rust_file: str
rust_test: str
reason: str
@dataclass(frozen=True, slots=True)
class TestLedger:
sdk_function: str
python_scope: tuple[str, ...]
rust_scope: tuple[str, ...]
entries: tuple[LedgerEntry, ...]
rust_only_tests: tuple[RustOnlyEntry, ...]
@property
def mapped_count(self) -> int:
return sum(1 for entry in self.entries if entry.status == "mapped")
@property
def total_count(self) -> int:
return len(self.entries)
@property
def percentage(self) -> float:
if self.total_count == 0:
return 0.0
return round(100.0 * self.mapped_count / self.total_count, 1)
def _require_string(value: Any, field: str, source: Path) -> str:
if not isinstance(value, str) or not value.strip():
raise ValueError(f"{source}: {field} must be a non-empty string")
return value
def _require_string_list(value: Any, field: str, source: Path) -> tuple[str, ...]:
if not isinstance(value, list) or not all(isinstance(item, str) and item for item in value):
raise ValueError(f"{source}: {field} must be a list of non-empty strings")
return tuple(value)
def _load_entry(data: Any, index: int, source: Path) -> LedgerEntry:
if not isinstance(data, dict):
raise ValueError(f"{source}: entries[{index}] must be an object")
python_file = _require_string(data.get("python_file"), f"entries[{index}].python_file", source)
python_test = _require_string(data.get("python_test"), f"entries[{index}].python_test", source)
status = data.get("status")
if status not in ("mapped", "unmapped"):
raise ValueError(f"{source}: entries[{index}].status must be 'mapped' or 'unmapped'")
if status == "mapped":
rust_file = _require_string(data.get("rust_file"), f"entries[{index}].rust_file", source)
rust_test = _require_string(data.get("rust_test"), f"entries[{index}].rust_test", source)
justification = _require_string(
data.get("justification"), f"entries[{index}].justification", source
)
return LedgerEntry(
python_file=python_file,
python_test=python_test,
status=status,
rust_file=rust_file,
rust_test=rust_test,
justification=justification,
reason="",
)
reason = _require_string(data.get("reason"), f"entries[{index}].reason", source)
return LedgerEntry(
python_file=python_file,
python_test=python_test,
status=status,
rust_file="",
rust_test="",
justification="",
reason=reason,
)
def _load_rust_only_entry(data: Any, index: int, source: Path) -> RustOnlyEntry:
if not isinstance(data, dict):
raise ValueError(f"{source}: rust_only_tests[{index}] must be an object")
return RustOnlyEntry(
rust_file=_require_string(data.get("rust_file"), f"rust_only_tests[{index}].rust_file", source),
rust_test=_require_string(data.get("rust_test"), f"rust_only_tests[{index}].rust_test", source),
reason=_require_string(data.get("reason"), f"rust_only_tests[{index}].reason", source),
)
def load_ledger(path: Path) -> TestLedger:
with path.open(encoding="utf-8") as stream:
data = json.load(stream)
sdk_function = _require_string(data.get("sdk_function"), "sdk_function", path)
python_scope = _require_string_list(data.get("python_scope"), "python_scope", path)
rust_scope = _require_string_list(data.get("rust_scope"), "rust_scope", path)
entries_data = data.get("entries")
if not isinstance(entries_data, list):
raise ValueError(f"{path}: entries must be a list")
entries = tuple(
_load_entry(entry, index, path) for index, entry in enumerate(entries_data)
)
rust_only_data = data.get("rust_only_tests")
if not isinstance(rust_only_data, list):
raise ValueError(f"{path}: rust_only_tests must be a list")
rust_only_tests = tuple(
_load_rust_only_entry(entry, index, path) for index, entry in enumerate(rust_only_data)
)
return TestLedger(
sdk_function=sdk_function,
python_scope=python_scope,
rust_scope=rust_scope,
entries=entries,
rust_only_tests=rust_only_tests,
)