mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-16 23:41:43 +00:00
ee08c36fc0
* wip * refactor(tests): move sdk function tracing into rust python harness * dead code * fix: handle harness keyboard interrupts * refactor(tests): deduplicate rust python harness helpers * fix(harness): expose validated strategy choices * wip * refactor(harness): let strategies own parity reports * docs(harness): update strategy structure * refactor(harness): localize strategy report views * wip * fix(harness): satisfy mapping runner type checks * fix(harness): clarify trace parity output * wip * fix(harness): clarify unit mapping report * fix(harness): finalize trace parity contracts * refactor(harness): structure parity contracts * feat: derive unit test mapping from traces * feat(harness): map rstest test families * feat(ocr): port Azure document intelligence tests * feat(harness): enforce complete unit mappings * feat(ocr): add reducto core transforms * feat(harness): classify host-only unit tests * fix(ocr): complete Rust provider plumbing * fix(harness): reuse OCR parity workers
84 lines
3.1 KiB
Python
84 lines
3.1 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
from typing import Final
|
|
|
|
from ...shared.reporting.models import Coverage, HarnessCase, HarnessRun, RunStatus
|
|
from ...shared.reporting.strategy import SuiteCaseSpec
|
|
from ...shared.unit_runners.suite_runner import run_suites
|
|
from .runner import UnitParityExclusion, UnitParitySuite, run_suite
|
|
|
|
|
|
def _write_tests(tmp_path: Path, *, mismatch: bool = False, failing: bool = False) -> None:
|
|
(tmp_path / "pytest.ini").write_text("[pytest]\n")
|
|
(tmp_path / "test_api.py").write_text(
|
|
"import os\n"
|
|
"def test_decode():\n assert int('42') == 42\n"
|
|
+ ("def test_backend():\n assert os.environ['LITELLM_RUST'] == '0'\n" if mismatch else "")
|
|
+ ("def test_fails():\n assert False\n" if failing else "")
|
|
)
|
|
|
|
|
|
def _case() -> HarnessCase:
|
|
return HarnessCase(
|
|
strategy_id="unit_tests_parity",
|
|
strategy_label="Unit test parity",
|
|
sdk_function="ocr",
|
|
spec=SuiteCaseSpec(coverage=Coverage.COMPLETE, suite="ocr"),
|
|
)
|
|
|
|
|
|
def _run(case: HarnessCase, tmp_path: Path, suite: UnitParitySuite) -> tuple[int, HarnessRun]:
|
|
return run_suites((case,), tmp_path, lambda _: None, suites={"ocr": suite}, execute=run_suite)
|
|
|
|
|
|
def test_passes_when_both_backends_agree(tmp_path: Path) -> None:
|
|
_write_tests(tmp_path)
|
|
case: Final = _case()
|
|
|
|
code, report = _run(case, tmp_path, UnitParitySuite(python_selectors=("test_api.py",)))
|
|
|
|
assert code == 0, report.failures
|
|
assert report.results[case.key].status is RunStatus.PASSED
|
|
|
|
|
|
def test_passes_when_both_backends_fail_identically(tmp_path: Path) -> None:
|
|
_write_tests(tmp_path, failing=True)
|
|
case: Final = _case()
|
|
|
|
code, report = _run(case, tmp_path, UnitParitySuite(python_selectors=("test_api.py",)))
|
|
|
|
assert code == 0, report.failures
|
|
assert report.results[case.key].status is RunStatus.PASSED
|
|
|
|
|
|
def test_fails_when_backend_outcomes_differ(tmp_path: Path) -> None:
|
|
_write_tests(tmp_path, mismatch=True)
|
|
case: Final = _case()
|
|
|
|
code, report = _run(case, tmp_path, UnitParitySuite(python_selectors=("test_api.py",)))
|
|
|
|
assert code == 1
|
|
assert report.results[case.key].status is RunStatus.FAILED
|
|
assert any("Python/Rust test outcomes differ" in detail for _, detail in report.failures)
|
|
assert any("Python only: test_api.py::test_backend [call] passed" in detail for _, detail in report.failures)
|
|
assert any("Rust only: test_api.py::test_backend [call] failed" in detail for _, detail in report.failures)
|
|
|
|
|
|
def test_excludes_tests_whose_contract_is_the_backend_flag(tmp_path: Path) -> None:
|
|
_write_tests(tmp_path, mismatch=True)
|
|
suite: Final = UnitParitySuite(
|
|
python_selectors=("test_api.py",),
|
|
exclusions=(
|
|
UnitParityExclusion(
|
|
nodeid="test_api.py::test_backend",
|
|
reason="The test intentionally asserts which backend is selected.",
|
|
),
|
|
),
|
|
)
|
|
case: Final = _case()
|
|
|
|
code, report = _run(case, tmp_path, suite)
|
|
|
|
assert code == 0, report.failures
|
|
assert report.results[case.key].status is RunStatus.PASSED
|