mirror of
https://github.com/BerriAI/litellm.git
synced 2026-09-09 22:31:41 +00:00
test: add regression coverage for the OCR ledger and audit script
Covers schema validation, AST/regex test enumeration, drift detection on both the Python and Rust sides, LedgerDriftError content, and a live-repo clean-audit guard against future drift.
This commit is contained in:
Ishaan Jaff
parent
a009ba4bd1
commit
6c88e5d7c9
1 changed files with 214 additions and 0 deletions
|
|
@ -8,11 +8,21 @@ import pytest
|
|||
|
||||
catalog = importlib.import_module("tests.rust-python-harness.catalog")
|
||||
cli = importlib.import_module("tests.rust-python-harness.cli")
|
||||
ledger_module = importlib.import_module("tests.rust-python-harness.ledger")
|
||||
models = importlib.import_module("tests.rust-python-harness.models")
|
||||
runner = importlib.import_module("tests.rust-python-harness.runner")
|
||||
ui = importlib.import_module("tests.rust-python-harness.ui")
|
||||
validate_ledger = importlib.import_module("tests.rust-python-harness.validate_ledger")
|
||||
|
||||
load_catalog = catalog.load_catalog
|
||||
load_ledger = ledger_module.load_ledger
|
||||
LEDGER_PATH = ledger_module.LEDGER_PATH
|
||||
REPO_ROOT = validate_ledger.REPO_ROOT
|
||||
audit_ledger = validate_ledger.audit_ledger
|
||||
run_audit = validate_ledger.run_audit
|
||||
LedgerDriftError = validate_ledger.LedgerDriftError
|
||||
_enumerate_python_tests = validate_ledger._enumerate_python_tests
|
||||
_enumerate_rust_tests = validate_ledger._enumerate_rust_tests
|
||||
_pick_values = cli._pick_values
|
||||
_coverage_pytest_args = cli._coverage_pytest_args
|
||||
_select = cli._select
|
||||
|
|
@ -234,3 +244,207 @@ def test_should_report_confidence_for_each_sdk_section() -> None:
|
|||
assert scores["responses"].level.value == "MEDIUM"
|
||||
assert scores["count_tokens"].percentage == 0
|
||||
assert scores["count_tokens"].level.value == "LOW"
|
||||
|
||||
|
||||
def test_should_load_the_ocr_ledger_and_compute_its_ratio() -> None:
|
||||
ledger = load_ledger()
|
||||
|
||||
assert ledger.sdk_function == "ocr"
|
||||
assert ledger.total_count == 145
|
||||
assert ledger.mapped_count == 8
|
||||
assert ledger.percentage == 5.5
|
||||
|
||||
|
||||
def test_should_reject_a_mapped_entry_missing_rust_test(tmp_path: Path) -> None:
|
||||
ledger_path = tmp_path / "bad_ledger.json"
|
||||
ledger_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"sdk_function": "ocr",
|
||||
"python_scope": ["tests/test_litellm/ocr/test_x.py"],
|
||||
"rust_scope": ["litellm-rust/crates/x.rs"],
|
||||
"entries": [
|
||||
{
|
||||
"python_file": "tests/test_litellm/ocr/test_x.py",
|
||||
"python_test": "test_one",
|
||||
"status": "mapped",
|
||||
"rust_file": "litellm-rust/crates/x.rs",
|
||||
}
|
||||
],
|
||||
"rust_only_tests": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="rust_test"):
|
||||
load_ledger(ledger_path)
|
||||
|
||||
|
||||
def test_should_reject_an_unmapped_entry_missing_a_reason(tmp_path: Path) -> None:
|
||||
ledger_path = tmp_path / "bad_ledger.json"
|
||||
ledger_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"sdk_function": "ocr",
|
||||
"python_scope": ["tests/test_litellm/ocr/test_x.py"],
|
||||
"rust_scope": [],
|
||||
"entries": [
|
||||
{
|
||||
"python_file": "tests/test_litellm/ocr/test_x.py",
|
||||
"python_test": "test_one",
|
||||
"status": "unmapped",
|
||||
}
|
||||
],
|
||||
"rust_only_tests": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="reason"):
|
||||
load_ledger(ledger_path)
|
||||
|
||||
|
||||
def test_should_enumerate_module_level_and_class_scoped_python_tests(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
module = tmp_path / "tests" / "example"
|
||||
module.mkdir(parents=True)
|
||||
source_path = module / "test_example.py"
|
||||
source_path.write_text(
|
||||
"def test_top_level():\n"
|
||||
" pass\n"
|
||||
"\n"
|
||||
"class TestGroup:\n"
|
||||
" def test_one(self):\n"
|
||||
" pass\n"
|
||||
"\n"
|
||||
" def test_two(self):\n"
|
||||
" pass\n"
|
||||
"\n"
|
||||
" def helper(self):\n"
|
||||
" pass\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
found = _enumerate_python_tests(tmp_path, "tests/example/test_example.py")
|
||||
|
||||
assert found == frozenset(
|
||||
{"test_top_level", "TestGroup::test_one", "TestGroup::test_two"}
|
||||
)
|
||||
|
||||
|
||||
def test_should_enumerate_rust_test_and_tokio_test_functions(tmp_path: Path) -> None:
|
||||
source_path = tmp_path / "example.rs"
|
||||
source_path.write_text(
|
||||
"#[test]\n"
|
||||
"fn plain_test() {\n"
|
||||
" assert!(true);\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"#[tokio::test]\n"
|
||||
"async fn async_test() {\n"
|
||||
" assert!(true);\n"
|
||||
"}\n"
|
||||
"\n"
|
||||
"fn not_a_test() {}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
found = _enumerate_rust_tests(tmp_path, "example.rs")
|
||||
|
||||
assert found == frozenset({"plain_test", "async_test"})
|
||||
|
||||
|
||||
def test_should_flag_a_python_test_missing_from_the_ledger(tmp_path: Path) -> None:
|
||||
(tmp_path / "litellm-rust").mkdir()
|
||||
python_dir = tmp_path / "tests" / "example"
|
||||
python_dir.mkdir(parents=True)
|
||||
(python_dir / "test_example.py").write_text(
|
||||
"def test_tracked():\n pass\n\n\ndef test_untracked():\n pass\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
ledger = load_ledger(_write_ledger(tmp_path, mapped_python_test="test_tracked"))
|
||||
|
||||
report = audit_ledger(ledger, repo_root=tmp_path)
|
||||
|
||||
assert report.stale_python_tests == ("tests/example/test_example.py:test_untracked",)
|
||||
assert not report.is_clean
|
||||
|
||||
|
||||
def test_should_flag_a_stale_ledger_entry_for_a_removed_python_test(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
(tmp_path / "litellm-rust").mkdir()
|
||||
python_dir = tmp_path / "tests" / "example"
|
||||
python_dir.mkdir(parents=True)
|
||||
(python_dir / "test_example.py").write_text(
|
||||
"def test_still_here():\n pass\n", encoding="utf-8"
|
||||
)
|
||||
ledger = load_ledger(_write_ledger(tmp_path, mapped_python_test="test_removed"))
|
||||
|
||||
report = audit_ledger(ledger, repo_root=tmp_path)
|
||||
|
||||
assert report.missing_python_tests == (
|
||||
"tests/example/test_example.py:test_removed",
|
||||
)
|
||||
assert not report.is_clean
|
||||
|
||||
|
||||
def test_should_raise_ledger_drift_error_with_every_offending_test(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
(tmp_path / "litellm-rust").mkdir()
|
||||
python_dir = tmp_path / "tests" / "example"
|
||||
python_dir.mkdir(parents=True)
|
||||
(python_dir / "test_example.py").write_text(
|
||||
"def test_still_here():\n pass\n", encoding="utf-8"
|
||||
)
|
||||
ledger_path = _write_ledger(tmp_path, mapped_python_test="test_removed")
|
||||
|
||||
with pytest.raises(LedgerDriftError, match="test_removed"):
|
||||
run_audit(ledger_path, repo_root=tmp_path)
|
||||
|
||||
|
||||
def _write_ledger(tmp_path: Path, *, mapped_python_test: str) -> Path:
|
||||
rust_dir = tmp_path / "litellm-rust"
|
||||
(rust_dir / "example.rs").write_text(
|
||||
'#[test]\nfn rust_side() {\n assert!(true);\n}\n', encoding="utf-8"
|
||||
)
|
||||
ledger_path = tmp_path / "ledger.json"
|
||||
ledger_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"sdk_function": "ocr",
|
||||
"python_scope": ["tests/example/test_example.py"],
|
||||
"rust_scope": ["litellm-rust/example.rs"],
|
||||
"entries": [
|
||||
{
|
||||
"python_file": "tests/example/test_example.py",
|
||||
"python_test": mapped_python_test,
|
||||
"status": "mapped",
|
||||
"rust_file": "litellm-rust/example.rs",
|
||||
"rust_test": "rust_side",
|
||||
"justification": "example",
|
||||
}
|
||||
],
|
||||
"rust_only_tests": [],
|
||||
}
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return ledger_path
|
||||
|
||||
|
||||
def test_should_report_the_real_ocr_ledger_as_clean_against_the_live_repo() -> None:
|
||||
ledger = load_ledger()
|
||||
|
||||
report = audit_ledger(ledger, repo_root=REPO_ROOT)
|
||||
|
||||
assert report.is_clean, (
|
||||
f"missing_python_tests={report.missing_python_tests} "
|
||||
f"stale_python_tests={report.stale_python_tests} "
|
||||
f"missing_rust_tests={report.missing_rust_tests} "
|
||||
f"stale_rust_tests={report.stale_rust_tests}"
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue