diff --git a/tests/test_litellm/_fixture_recorder.py b/tests/test_litellm/_fixture_recorder.py index a2d4ed06bbc..849efd15040 100644 --- a/tests/test_litellm/_fixture_recorder.py +++ b/tests/test_litellm/_fixture_recorder.py @@ -12,6 +12,7 @@ from pathlib import Path from typing import Final, cast import httpx +from pydantic import ValidationError from tests.test_litellm._json_fs_cache import JsonFileCache, canonical_json from tests.test_litellm.ocr.fixture_models import ( @@ -194,7 +195,17 @@ def fixture_directory(configured: Path | None, env_value: str | None, default: P def recorded_fixtures(directory: Path) -> tuple[OcrParityCase, ...]: - return tuple(OcrParityCase.model_validate(raw_fixture) for raw_fixture in JsonFileCache(directory).values()) + cache: Final = JsonFileCache(directory) + fixtures: list[OcrParityCase] = [] + for path, raw_fixture in cache.values_with_paths(): + try: + fixtures.append(OcrParityCase.model_validate(raw_fixture)) + except ValidationError as error: + raise ValueError( + f"invalid OCR parity fixture {path}: expected exactly `input` and `upstream_response` " + f"({len(error.errors())} validation errors)" + ) from error + return tuple(fixtures) def fixture_id(fixture: OcrParityCase) -> str: diff --git a/tests/test_litellm/_json_fs_cache.py b/tests/test_litellm/_json_fs_cache.py index 9ce43931e36..dfc8b6331f4 100644 --- a/tests/test_litellm/_json_fs_cache.py +++ b/tests/test_litellm/_json_fs_cache.py @@ -43,3 +43,9 @@ class JsonFileCache: return () paths: Final = tuple(sorted(self.root.rglob("*.json"))) return tuple(JSON_OBJECT.validate_json(path.read_text(encoding="utf-8")) for path in paths) + + def values_with_paths(self) -> tuple[tuple[Path, dict[str, object]], ...]: + if not self.root.is_dir(): + return () + paths: Final = tuple(sorted(self.root.rglob("*.json"))) + return tuple((path, JSON_OBJECT.validate_json(path.read_text(encoding="utf-8"))) for path in paths) diff --git a/tests/test_litellm/ocr/conftest.py b/tests/test_litellm/ocr/conftest.py index 2822efe23ad..b5410325905 100644 --- a/tests/test_litellm/ocr/conftest.py +++ b/tests/test_litellm/ocr/conftest.py @@ -5,6 +5,7 @@ from pathlib import Path from typing import Final import pytest +from pydantic import ValidationError from tests.test_litellm._fixture_recorder import fixture_id, recorded_fixtures @@ -23,16 +24,26 @@ def _fixture_directory() -> Path: def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: if "ocr_fixture" not in metafunc.fixturenames: return - fixtures: Final = recorded_fixtures(_fixture_directory()) + directory: Final = _fixture_directory() + try: + fixtures: Final = recorded_fixtures(directory) + except (ValidationError, ValueError) as error: + raise pytest.UsageError( + f"Invalid OCR parity fixture bundle at {directory}. " + "Each fixture must contain exactly `input` and `upstream_response`. " + "Record fresh fixtures in an empty directory with: " + f"`uv run python tests/test_litellm/ocr/generate_fixtures.py --fixture-dir {directory}`. " + f"Validation details: {error}" + ) from error if not fixtures: if FIXTURE_DIR_ENV in os.environ: - raise pytest.UsageError(f"no recorded OCR fixtures in {_fixture_directory()}") + raise pytest.UsageError(f"no recorded OCR fixtures in {directory}") metafunc.parametrize( "ocr_fixture", ( pytest.param( None, - marks=pytest.mark.skip(reason=f"no recorded OCR fixtures in {_fixture_directory()}"), + marks=pytest.mark.skip(reason=f"no recorded OCR fixtures in {directory}"), id="no-recorded-fixtures", ), ),