refactor(tests): align parity harness layout

.github/workflows/test-unit.yml

@@ -106,7 +106,7 @@ jobs:
               tests/test_litellm/endpoints
               tests/test_litellm/experimental_mcp_client
               tests/test_litellm/models
-              tests/test_litellm/parity
+              tests/route_parity
               tests/test_litellm/repositories
               tests/test_litellm/images
               tests/test_litellm/interactions

tests/route_parity/__init__.py

@@ -0,0 +1,3 @@
+import pytest
+
+pytest.register_assert_rewrite("tests.route_parity.compare")

tests/route_parity/compare.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 from typing import Final
 
-from tests.test_litellm.parity.models import CapturedRequest, Execution
+from tests.route_parity.models import CapturedRequest, Execution
 
 
 def validate_harness(python: Execution, accelerated: Execution, python_user_agent: str) -> None:

tests/route_parity/fixture_generator.py

@@ -10,8 +10,8 @@ from typing import Final, Generic, TypeVar, cast
 from hypothesis.strategies import SearchStrategy
 from pydantic import BaseModel
 
-from tests.test_litellm._fixture_models import SdkInputBase
-from tests.test_litellm._fixture_recorder import ProviderSpec, generate_case_inputs, record_cases
+from tests.route_parity.fixture_models import SdkInputBase
+from tests.route_parity.fixture_recorder import ProviderSpec, generate_case_inputs, record_cases
 
 LOGGER: Final = logging.getLogger(__name__)
 InputT = TypeVar("InputT", bound=SdkInputBase)
@@ -57,7 +57,9 @@ def generate_target_fixtures(
         )
 
 
-def require_targets(targets: tuple[FixtureTarget[InputT], ...], error_message: str) -> tuple[FixtureTarget[InputT], ...]:
+def require_targets(
+    targets: tuple[FixtureTarget[InputT], ...], error_message: str
+) -> tuple[FixtureTarget[InputT], ...]:
     if targets:
         return targets
     raise SystemExit(error_message)

tests/route_parity/fixture_models.py

@@ -4,7 +4,7 @@ from typing import Generic, Literal, TypeVar, cast
 
 from pydantic import BaseModel, ConfigDict, Field, JsonValue
 
-from tests.test_litellm._recorded_http import RecordedResponse
+from tests.route_parity.recorded_http import RecordedResponse
 
 JsonObject = dict[str, JsonValue]
 

tests/route_parity/fixture_recorder.py

@@ -19,8 +19,8 @@ from hypothesis import given, settings
 from hypothesis.strategies import SearchStrategy
 from pydantic import AwareDatetime, BaseModel, ConfigDict, ValidationError
 
-from tests.test_litellm._json_fs_cache import JsonFileCache, canonical_json
-from tests.test_litellm._recorded_http import (
+from tests.route_parity.json_file_cache import JsonFileCache, canonical_json
+from tests.route_parity.recorded_http import (
     HttpHeader,
     RecordedHttpResponse,
     RecordedHttpStreamResponse,
@@ -134,7 +134,9 @@ class _RecordingHandler(BaseHTTPRequestHandler):
         if 200 <= recorded_response.status_code < 300:
             provider.responses.put(recorded_response)
         if isinstance(recorded_response, RecordedHttpResponse):
-            self._send_response(recorded_response.status_code, recorded_response.headers, recorded_response.body_bytes())
+            self._send_response(
+                recorded_response.status_code, recorded_response.headers, recorded_response.body_bytes()
+            )
 
     def _record_upstream_response(
         self,

tests/route_parity/models.py

@@ -4,6 +4,7 @@ from typing import Annotated, Literal
 
 from pydantic import BaseModel, ConfigDict, Field, JsonValue
 
+
 class CapturedRequest(BaseModel):
     model_config = ConfigDict(frozen=True)
 

tests/route_parity/replay.py

@@ -9,8 +9,8 @@ from typing import Final
 
 from pydantic import JsonValue, TypeAdapter
 
-from tests.test_litellm._recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse
-from tests.test_litellm.parity.models import CapturedRequest
+from tests.route_parity.models import CapturedRequest
+from tests.route_parity.recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse
 
 JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue)
 EXCLUDED_REQUEST_HEADERS: Final = frozenset(

tests/route_parity/runner.py

@@ -14,15 +14,15 @@ from typing import Final, TextIO, cast
 
 from pydantic import TypeAdapter, ValidationError
 
-from tests.test_litellm._recorded_http import RecordedResponse
-from tests.test_litellm.parity.models import (
+from tests.route_parity.models import (
     Execution,
     SDKCommand,
     WorkerFailure,
     WorkerResult,
     WorkerSuccess,
 )
-from tests.test_litellm.parity.replay import ReplayServer, replay_server
+from tests.route_parity.recorded_http import RecordedResponse
+from tests.route_parity.replay import ReplayServer, replay_server
 
 WORKER_RESULT_PREFIX: Final = "LITELLM_PARITY_RESULT "
 WORKER_RESULT_ADAPTER: Final[TypeAdapter[WorkerResult]] = TypeAdapter(WorkerResult)
@@ -181,9 +181,7 @@ def parity_worker_main(
     event_loop: Final = asyncio.new_event_loop()
     try:
         for line in sys.stdin:
-            sys.stdout.write(
-                f"{WORKER_RESULT_PREFIX}{execute_command(line, mock_url, event_loop).model_dump_json()}\n"
-            )
+            sys.stdout.write(f"{WORKER_RESULT_PREFIX}{execute_command(line, mock_url, event_loop).model_dump_json()}\n")
             sys.stdout.flush()
     finally:
         event_loop.close()

tests/route_parity/test_fixture_recorder.py

@@ -12,7 +12,7 @@ import pytest
 from hypothesis import strategies as st
 from pydantic import BaseModel, ConfigDict
 
-from tests.test_litellm._fixture_recorder import (
+from tests.route_parity.fixture_recorder import (
     FIXTURE_SCHEMA_VERSION,
     ProviderSpec,
     fixture_cache_key,
@@ -21,14 +21,14 @@ from tests.test_litellm._fixture_recorder import (
     record_cases,
     recorded_fixtures,
 )
-from tests.test_litellm._json_fs_cache import JsonFileCache
-from tests.test_litellm._recorded_http import (
+from tests.route_parity.json_file_cache import JsonFileCache
+from tests.route_parity.recorded_http import (
     HttpHeader,
     RecordedHttpStreamResponse,
     RecordedResponse,
     RecordedStreamChunk,
 )
-from tests.test_litellm.parity.replay import replay_server
+from tests.route_parity.replay import replay_server
 
 _SSE_CHUNKS: Final = (
     b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n',

tests/route_parity/test_parity.py

@@ -5,8 +5,8 @@ from typing import Final
 import pytest
 from pydantic import JsonValue
 
-from tests.test_litellm.parity.compare import assert_parity
-from tests.test_litellm.parity.models import CapturedRequest, Execution, SDKReport
+from tests.route_parity.compare import assert_parity
+from tests.route_parity.models import CapturedRequest, Execution, SDKReport
 
 SENTINEL: Final = "python-parity-fallback"
 
@@ -20,9 +20,7 @@ def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: st
             body={"model": "test-model"} if body is None else body,
             user_agent=user_agent,
         ),
-        report=SDKReport(
-            response={"items": [{"text": markdown}], "model": "test-model"}
-        ),
+        report=SDKReport(response={"items": [{"text": markdown}], "model": "test-model"}),
     )
 
 

tests/test_litellm/ocr/conftest.py

@@ -5,7 +5,7 @@ from typing import Final
 
 import pytest
 
-from tests.test_litellm._fixture_recorder import fixture_id, parametrize_recorded_fixtures
+from tests.route_parity.fixture_recorder import fixture_id, parametrize_recorded_fixtures
 from tests.test_litellm.ocr.fixture_models import OcrParityCase
 
 FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR"
@@ -20,7 +20,7 @@ def _fixture_id(fixture: OcrParityCase) -> str:
 
 
 def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
-    default_directory: Final = Path(__file__).with_name(".fixtures")
+    default_directory: Final = Path(__file__).with_name("fixtures")
     parametrize_recorded_fixtures(
         metafunc,
         fixture_name="ocr_fixture",
@@ -28,8 +28,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
         env_var=FIXTURE_DIR_ENV,
         default_directory=default_directory,
         regeneration_command=(
-            "uv run python tests/test_litellm/ocr/generate_fixtures.py "
-            f"--fixture-dir {default_directory}"
+            f"uv run python tests/test_litellm/ocr/generate_fixtures.py --fixture-dir {default_directory}"
         ),
         id_builder=_fixture_id,
     )

tests/test_litellm/ocr/fixture_models.py

@@ -7,7 +7,7 @@ from typing import Annotated, Literal
 from pydantic import Field, field_validator, model_validator
 from typing_extensions import Self
 
-from tests.test_litellm._fixture_models import (
+from tests.route_parity.fixture_models import (
     FixtureModel,
     JsonObject,
     JsonSchemaDefinition,

tests/test_litellm/ocr/generate_fixtures.py

@@ -13,15 +13,15 @@ from hypothesis.strategies import DrawFn, SearchStrategy
 
 import litellm
 from litellm.rust_bridge.ocr import use_litellm_rust
-from tests.test_litellm._fixture_generator import (
+from tests.route_parity.fixture_generator import (
     FixtureTarget,
     generate_target_fixtures,
     parse_generator_args,
 )
-from tests.test_litellm._fixture_generator import (
+from tests.route_parity.fixture_generator import (
     require_targets as require_fixture_targets,
 )
-from tests.test_litellm._fixture_recorder import (
+from tests.route_parity.fixture_recorder import (
     ProviderSpec,
     fixture_directory,
 )
@@ -291,7 +291,7 @@ def main() -> None:
     root: Final = fixture_directory(
         args.fixture_dir,
         os.environ.get(FIXTURE_DIR_ENV),
-        Path(__file__).with_name(".fixtures"),
+        Path(__file__).with_name("fixtures"),
     )
     use_litellm_rust(False, ocr=None, aocr=None)
     for target in targets:

tests/test_litellm/ocr/test_generate_fixtures.py

@@ -6,7 +6,7 @@ from typing import Final
 
 import pytest
 
-from tests.test_litellm._fixture_recorder import generate_case_inputs
+from tests.route_parity.fixture_recorder import generate_case_inputs
 from tests.test_litellm.ocr.generate_fixtures import (
     discover_targets,
     parse_generator_args,

tests/test_litellm/ocr/test_sdk_parity.py

@@ -11,16 +11,16 @@ from typing import Final, cast
 import pytest
 
 from litellm.llms.base_llm.ocr.transformation import OCRResponse
-from tests.test_litellm.ocr.fixture_models import MistralOcrSdkInput, OcrParityCase
-from tests.test_litellm.parity.compare import assert_parity
-from tests.test_litellm.parity.models import SDKCommand, SDKReport, WorkerFailure, WorkerResult, WorkerSuccess
-from tests.test_litellm.parity.runner import (
+from tests.route_parity.compare import assert_parity
+from tests.route_parity.models import SDKCommand, SDKReport, WorkerFailure, WorkerResult, WorkerSuccess
+from tests.route_parity.runner import (
     PythonScriptRunner,
     PythonScriptWorker,
     execution_worker_pair,
     parity_worker_main,
     run_execution,
 )
+from tests.test_litellm.ocr.fixture_models import MistralOcrSdkInput, OcrParityCase
 
 API_KEY: Final = "test-key"
 PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback"

tests/test_litellm/parity/__init__.py

@@ -1,3 +0,0 @@
-import pytest
-
-pytest.register_assert_rewrite("tests.test_litellm.parity.compare")