diff --git a/.github/workflows/test-unit.yml b/.github/workflows/test-unit.yml index 0b5c648c13d..fff77acf5f9 100644 --- a/.github/workflows/test-unit.yml +++ b/.github/workflows/test-unit.yml @@ -106,7 +106,7 @@ jobs: tests/test_litellm/endpoints tests/test_litellm/experimental_mcp_client tests/test_litellm/models - tests/test_litellm/parity + tests/route_parity tests/test_litellm/repositories tests/test_litellm/images tests/test_litellm/interactions diff --git a/tests/route_parity/__init__.py b/tests/route_parity/__init__.py new file mode 100644 index 00000000000..df2a40db636 --- /dev/null +++ b/tests/route_parity/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.register_assert_rewrite("tests.route_parity.compare") diff --git a/tests/test_litellm/parity/compare.py b/tests/route_parity/compare.py similarity index 94% rename from tests/test_litellm/parity/compare.py rename to tests/route_parity/compare.py index 7386684c331..e76079b6cb9 100644 --- a/tests/test_litellm/parity/compare.py +++ b/tests/route_parity/compare.py @@ -2,7 +2,7 @@ from __future__ import annotations from typing import Final -from tests.test_litellm.parity.models import CapturedRequest, Execution +from tests.route_parity.models import CapturedRequest, Execution def validate_harness(python: Execution, accelerated: Execution, python_user_agent: str) -> None: diff --git a/tests/test_litellm/_fixture_generator.py b/tests/route_parity/fixture_generator.py similarity index 87% rename from tests/test_litellm/_fixture_generator.py rename to tests/route_parity/fixture_generator.py index 40b9f0f9777..eeef76eb264 100644 --- a/tests/test_litellm/_fixture_generator.py +++ b/tests/route_parity/fixture_generator.py @@ -10,8 +10,8 @@ from typing import Final, Generic, TypeVar, cast from hypothesis.strategies import SearchStrategy from pydantic import BaseModel -from tests.test_litellm._fixture_models import SdkInputBase -from tests.test_litellm._fixture_recorder import ProviderSpec, generate_case_inputs, record_cases +from tests.route_parity.fixture_models import SdkInputBase +from tests.route_parity.fixture_recorder import ProviderSpec, generate_case_inputs, record_cases LOGGER: Final = logging.getLogger(__name__) InputT = TypeVar("InputT", bound=SdkInputBase) @@ -57,7 +57,9 @@ def generate_target_fixtures( ) -def require_targets(targets: tuple[FixtureTarget[InputT], ...], error_message: str) -> tuple[FixtureTarget[InputT], ...]: +def require_targets( + targets: tuple[FixtureTarget[InputT], ...], error_message: str +) -> tuple[FixtureTarget[InputT], ...]: if targets: return targets raise SystemExit(error_message) diff --git a/tests/test_litellm/_fixture_models.py b/tests/route_parity/fixture_models.py similarity index 94% rename from tests/test_litellm/_fixture_models.py rename to tests/route_parity/fixture_models.py index a9aecf0730a..799cb472da4 100644 --- a/tests/test_litellm/_fixture_models.py +++ b/tests/route_parity/fixture_models.py @@ -4,7 +4,7 @@ from typing import Generic, Literal, TypeVar, cast from pydantic import BaseModel, ConfigDict, Field, JsonValue -from tests.test_litellm._recorded_http import RecordedResponse +from tests.route_parity.recorded_http import RecordedResponse JsonObject = dict[str, JsonValue] diff --git a/tests/test_litellm/_fixture_recorder.py b/tests/route_parity/fixture_recorder.py similarity index 97% rename from tests/test_litellm/_fixture_recorder.py rename to tests/route_parity/fixture_recorder.py index 18939f46ef5..6cb9141f9b5 100644 --- a/tests/test_litellm/_fixture_recorder.py +++ b/tests/route_parity/fixture_recorder.py @@ -19,8 +19,8 @@ from hypothesis import given, settings from hypothesis.strategies import SearchStrategy from pydantic import AwareDatetime, BaseModel, ConfigDict, ValidationError -from tests.test_litellm._json_fs_cache import JsonFileCache, canonical_json -from tests.test_litellm._recorded_http import ( +from tests.route_parity.json_file_cache import JsonFileCache, canonical_json +from tests.route_parity.recorded_http import ( HttpHeader, RecordedHttpResponse, RecordedHttpStreamResponse, @@ -134,7 +134,9 @@ class _RecordingHandler(BaseHTTPRequestHandler): if 200 <= recorded_response.status_code < 300: provider.responses.put(recorded_response) if isinstance(recorded_response, RecordedHttpResponse): - self._send_response(recorded_response.status_code, recorded_response.headers, recorded_response.body_bytes()) + self._send_response( + recorded_response.status_code, recorded_response.headers, recorded_response.body_bytes() + ) def _record_upstream_response( self, diff --git a/tests/test_litellm/_json_fs_cache.py b/tests/route_parity/json_file_cache.py similarity index 100% rename from tests/test_litellm/_json_fs_cache.py rename to tests/route_parity/json_file_cache.py diff --git a/tests/test_litellm/parity/models.py b/tests/route_parity/models.py similarity index 99% rename from tests/test_litellm/parity/models.py rename to tests/route_parity/models.py index a776936832e..e61cdf1ef84 100644 --- a/tests/test_litellm/parity/models.py +++ b/tests/route_parity/models.py @@ -4,6 +4,7 @@ from typing import Annotated, Literal from pydantic import BaseModel, ConfigDict, Field, JsonValue + class CapturedRequest(BaseModel): model_config = ConfigDict(frozen=True) diff --git a/tests/test_litellm/_recorded_http.py b/tests/route_parity/recorded_http.py similarity index 100% rename from tests/test_litellm/_recorded_http.py rename to tests/route_parity/recorded_http.py diff --git a/tests/test_litellm/parity/replay.py b/tests/route_parity/replay.py similarity index 95% rename from tests/test_litellm/parity/replay.py rename to tests/route_parity/replay.py index e9cf70e009a..d9eef1181a7 100644 --- a/tests/test_litellm/parity/replay.py +++ b/tests/route_parity/replay.py @@ -9,8 +9,8 @@ from typing import Final from pydantic import JsonValue, TypeAdapter -from tests.test_litellm._recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse -from tests.test_litellm.parity.models import CapturedRequest +from tests.route_parity.models import CapturedRequest +from tests.route_parity.recorded_http import RecordedHttpResponse, RecordedHttpStreamResponse, RecordedResponse JSON_VALUE: Final[TypeAdapter[JsonValue]] = TypeAdapter(JsonValue) EXCLUDED_REQUEST_HEADERS: Final = frozenset( diff --git a/tests/test_litellm/parity/runner.py b/tests/route_parity/runner.py similarity index 95% rename from tests/test_litellm/parity/runner.py rename to tests/route_parity/runner.py index d37c738627e..35fdf1689e0 100644 --- a/tests/test_litellm/parity/runner.py +++ b/tests/route_parity/runner.py @@ -14,15 +14,15 @@ from typing import Final, TextIO, cast from pydantic import TypeAdapter, ValidationError -from tests.test_litellm._recorded_http import RecordedResponse -from tests.test_litellm.parity.models import ( +from tests.route_parity.models import ( Execution, SDKCommand, WorkerFailure, WorkerResult, WorkerSuccess, ) -from tests.test_litellm.parity.replay import ReplayServer, replay_server +from tests.route_parity.recorded_http import RecordedResponse +from tests.route_parity.replay import ReplayServer, replay_server WORKER_RESULT_PREFIX: Final = "LITELLM_PARITY_RESULT " WORKER_RESULT_ADAPTER: Final[TypeAdapter[WorkerResult]] = TypeAdapter(WorkerResult) @@ -181,9 +181,7 @@ def parity_worker_main( event_loop: Final = asyncio.new_event_loop() try: for line in sys.stdin: - sys.stdout.write( - f"{WORKER_RESULT_PREFIX}{execute_command(line, mock_url, event_loop).model_dump_json()}\n" - ) + sys.stdout.write(f"{WORKER_RESULT_PREFIX}{execute_command(line, mock_url, event_loop).model_dump_json()}\n") sys.stdout.flush() finally: event_loop.close() diff --git a/tests/test_litellm/test__fixture_recorder.py b/tests/route_parity/test_fixture_recorder.py similarity index 97% rename from tests/test_litellm/test__fixture_recorder.py rename to tests/route_parity/test_fixture_recorder.py index dca494b0a22..86a95e6a7ae 100644 --- a/tests/test_litellm/test__fixture_recorder.py +++ b/tests/route_parity/test_fixture_recorder.py @@ -12,7 +12,7 @@ import pytest from hypothesis import strategies as st from pydantic import BaseModel, ConfigDict -from tests.test_litellm._fixture_recorder import ( +from tests.route_parity.fixture_recorder import ( FIXTURE_SCHEMA_VERSION, ProviderSpec, fixture_cache_key, @@ -21,14 +21,14 @@ from tests.test_litellm._fixture_recorder import ( record_cases, recorded_fixtures, ) -from tests.test_litellm._json_fs_cache import JsonFileCache -from tests.test_litellm._recorded_http import ( +from tests.route_parity.json_file_cache import JsonFileCache +from tests.route_parity.recorded_http import ( HttpHeader, RecordedHttpStreamResponse, RecordedResponse, RecordedStreamChunk, ) -from tests.test_litellm.parity.replay import replay_server +from tests.route_parity.replay import replay_server _SSE_CHUNKS: Final = ( b'data: {"choices":[{"delta":{"content":"hello"}}]}\n\n', diff --git a/tests/test_litellm/parity/test_parity.py b/tests/route_parity/test_parity.py similarity index 84% rename from tests/test_litellm/parity/test_parity.py rename to tests/route_parity/test_parity.py index 03de1d76f43..3deeec47032 100644 --- a/tests/test_litellm/parity/test_parity.py +++ b/tests/route_parity/test_parity.py @@ -5,8 +5,8 @@ from typing import Final import pytest from pydantic import JsonValue -from tests.test_litellm.parity.compare import assert_parity -from tests.test_litellm.parity.models import CapturedRequest, Execution, SDKReport +from tests.route_parity.compare import assert_parity +from tests.route_parity.models import CapturedRequest, Execution, SDKReport SENTINEL: Final = "python-parity-fallback" @@ -20,9 +20,7 @@ def _execution(*, body: JsonValue = None, markdown: str = "same", user_agent: st body={"model": "test-model"} if body is None else body, user_agent=user_agent, ), - report=SDKReport( - response={"items": [{"text": markdown}], "model": "test-model"} - ), + report=SDKReport(response={"items": [{"text": markdown}], "model": "test-model"}), ) diff --git a/tests/test_litellm/ocr/conftest.py b/tests/test_litellm/ocr/conftest.py index 367e2f02405..bfd657061c6 100644 --- a/tests/test_litellm/ocr/conftest.py +++ b/tests/test_litellm/ocr/conftest.py @@ -5,7 +5,7 @@ from typing import Final import pytest -from tests.test_litellm._fixture_recorder import fixture_id, parametrize_recorded_fixtures +from tests.route_parity.fixture_recorder import fixture_id, parametrize_recorded_fixtures from tests.test_litellm.ocr.fixture_models import OcrParityCase FIXTURE_DIR_ENV: Final = "LITELLM_OCR_FIXTURE_DIR" @@ -20,7 +20,7 @@ def _fixture_id(fixture: OcrParityCase) -> str: def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: - default_directory: Final = Path(__file__).with_name(".fixtures") + default_directory: Final = Path(__file__).with_name("fixtures") parametrize_recorded_fixtures( metafunc, fixture_name="ocr_fixture", @@ -28,8 +28,7 @@ def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: env_var=FIXTURE_DIR_ENV, default_directory=default_directory, regeneration_command=( - "uv run python tests/test_litellm/ocr/generate_fixtures.py " - f"--fixture-dir {default_directory}" + f"uv run python tests/test_litellm/ocr/generate_fixtures.py --fixture-dir {default_directory}" ), id_builder=_fixture_id, ) diff --git a/tests/test_litellm/ocr/fixture_models.py b/tests/test_litellm/ocr/fixture_models.py index 9494c3690b1..4257d517663 100644 --- a/tests/test_litellm/ocr/fixture_models.py +++ b/tests/test_litellm/ocr/fixture_models.py @@ -7,7 +7,7 @@ from typing import Annotated, Literal from pydantic import Field, field_validator, model_validator from typing_extensions import Self -from tests.test_litellm._fixture_models import ( +from tests.route_parity.fixture_models import ( FixtureModel, JsonObject, JsonSchemaDefinition, diff --git a/tests/test_litellm/ocr/.fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json b/tests/test_litellm/ocr/fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json similarity index 100% rename from tests/test_litellm/ocr/.fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json rename to tests/test_litellm/ocr/fixtures/067fe064938269ae58ac14030064a77b92d76edfd00e485fca93a0efbb82bda3.json diff --git a/tests/test_litellm/ocr/.fixtures/6aa18aa0bb41abea634cac9c77fcb73438844c5eb7b4f5eae1a86ecce7b60742.json b/tests/test_litellm/ocr/fixtures/6aa18aa0bb41abea634cac9c77fcb73438844c5eb7b4f5eae1a86ecce7b60742.json similarity index 100% rename from tests/test_litellm/ocr/.fixtures/6aa18aa0bb41abea634cac9c77fcb73438844c5eb7b4f5eae1a86ecce7b60742.json rename to tests/test_litellm/ocr/fixtures/6aa18aa0bb41abea634cac9c77fcb73438844c5eb7b4f5eae1a86ecce7b60742.json diff --git a/tests/test_litellm/ocr/.fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json b/tests/test_litellm/ocr/fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json similarity index 100% rename from tests/test_litellm/ocr/.fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json rename to tests/test_litellm/ocr/fixtures/d7a77e2b36580d60ba22ef0a9cd910adf782ab9194e309d3c62fe0f2fadc326c.json diff --git a/tests/test_litellm/ocr/.fixtures/ecfb251cbe6cb13e52dd11a4706f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json b/tests/test_litellm/ocr/fixtures/ecfb251cbe6cb13e52dd11a4706f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json similarity index 100% rename from tests/test_litellm/ocr/.fixtures/ecfb251cbe6cb13e52dd11a4706f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json rename to tests/test_litellm/ocr/fixtures/ecfb251cbe6cb13e52dd11a4706f788cf6f2b3d5f4fdae5d7b9b620c398f47dc.json diff --git a/tests/test_litellm/ocr/generate_fixtures.py b/tests/test_litellm/ocr/generate_fixtures.py index 531d44feb00..5ec498eca56 100644 --- a/tests/test_litellm/ocr/generate_fixtures.py +++ b/tests/test_litellm/ocr/generate_fixtures.py @@ -13,15 +13,15 @@ from hypothesis.strategies import DrawFn, SearchStrategy import litellm from litellm.rust_bridge.ocr import use_litellm_rust -from tests.test_litellm._fixture_generator import ( +from tests.route_parity.fixture_generator import ( FixtureTarget, generate_target_fixtures, parse_generator_args, ) -from tests.test_litellm._fixture_generator import ( +from tests.route_parity.fixture_generator import ( require_targets as require_fixture_targets, ) -from tests.test_litellm._fixture_recorder import ( +from tests.route_parity.fixture_recorder import ( ProviderSpec, fixture_directory, ) @@ -291,7 +291,7 @@ def main() -> None: root: Final = fixture_directory( args.fixture_dir, os.environ.get(FIXTURE_DIR_ENV), - Path(__file__).with_name(".fixtures"), + Path(__file__).with_name("fixtures"), ) use_litellm_rust(False, ocr=None, aocr=None) for target in targets: diff --git a/tests/test_litellm/ocr/test_generate_fixtures.py b/tests/test_litellm/ocr/test_generate_fixtures.py index cc9609cc10f..ec1a4509ff6 100644 --- a/tests/test_litellm/ocr/test_generate_fixtures.py +++ b/tests/test_litellm/ocr/test_generate_fixtures.py @@ -6,7 +6,7 @@ from typing import Final import pytest -from tests.test_litellm._fixture_recorder import generate_case_inputs +from tests.route_parity.fixture_recorder import generate_case_inputs from tests.test_litellm.ocr.generate_fixtures import ( discover_targets, parse_generator_args, diff --git a/tests/test_litellm/ocr/test_sdk_parity.py b/tests/test_litellm/ocr/test_sdk_parity.py index 7cc8c05f533..577d8b48522 100644 --- a/tests/test_litellm/ocr/test_sdk_parity.py +++ b/tests/test_litellm/ocr/test_sdk_parity.py @@ -11,16 +11,16 @@ from typing import Final, cast import pytest from litellm.llms.base_llm.ocr.transformation import OCRResponse -from tests.test_litellm.ocr.fixture_models import MistralOcrSdkInput, OcrParityCase -from tests.test_litellm.parity.compare import assert_parity -from tests.test_litellm.parity.models import SDKCommand, SDKReport, WorkerFailure, WorkerResult, WorkerSuccess -from tests.test_litellm.parity.runner import ( +from tests.route_parity.compare import assert_parity +from tests.route_parity.models import SDKCommand, SDKReport, WorkerFailure, WorkerResult, WorkerSuccess +from tests.route_parity.runner import ( PythonScriptRunner, PythonScriptWorker, execution_worker_pair, parity_worker_main, run_execution, ) +from tests.test_litellm.ocr.fixture_models import MistralOcrSdkInput, OcrParityCase API_KEY: Final = "test-key" PYTHON_HTTP_SENTINEL: Final = "python-ocr-parity-fallback" diff --git a/tests/test_litellm/parity/__init__.py b/tests/test_litellm/parity/__init__.py deleted file mode 100644 index b72fc0dc894..00000000000 --- a/tests/test_litellm/parity/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import pytest - -pytest.register_assert_rewrite("tests.test_litellm.parity.compare")