diff --git a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs index c2cb23d0010..fd59681655f 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/ocr/transformation.rs @@ -204,15 +204,27 @@ fn validate_location(location: &str) -> Result<(), Error> { #[cfg(test)] mod tests { + use rstest::rstest; + use super::VertexAiOcrConfig; - #[test] - fn endpoint_uses_location_project_and_model() { + /// The hosts `get_vertex_base_url` in `litellm/llms/vertex_ai/common_utils.py` picks. + #[rstest] + #[case::regional("europe-west4", "https://europe-west4-aiplatform.googleapis.com")] + #[case::global("global", "https://aiplatform.googleapis.com")] + #[case::multi_region_us("us", "https://aiplatform.us.rep.googleapis.com")] + #[case::multi_region_eu("eu", "https://aiplatform.eu.rep.googleapis.com")] + fn endpoint_uses_the_location_host_project_and_model( + #[case] location: &str, + #[case] host: &str, + ) { assert_eq!( VertexAiOcrConfig - .build_ocr_url(None, "proj-1", "europe-west4", "mistral-ocr-maas") + .build_ocr_url(None, "proj-1", location, "mistral-ocr-maas") .unwrap(), - "https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict" + format!( + "{host}/v1/projects/proj-1/locations/{location}/publishers/mistralai/models/mistral-ocr-maas:rawPredict" + ) ); } diff --git a/tests/test_litellm_rust/conftest.py b/tests/test_litellm_rust/conftest.py index 1b6fcfa00db..87b4cb1d4ae 100644 --- a/tests/test_litellm_rust/conftest.py +++ b/tests/test_litellm_rust/conftest.py @@ -41,6 +41,13 @@ async def isolate_ocr_test_state() -> AsyncIterator[None]: await GLOBAL_LOGGING_WORKER.stop() +@pytest.fixture(params=[False, True], ids=["python", "rust"]) +def ocr_backend(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> bool: + enabled: Final = bool(request.param) + monkeypatch.setenv("LITELLM_RUST", "1" if enabled else "0") + return enabled + + @pytest.fixture def recording_server() -> Generator[RecordingServer]: with recording_service() as server: diff --git a/tests/test_litellm_rust/ocr/conftest.py b/tests/test_litellm_rust/ocr/conftest.py new file mode 100644 index 00000000000..5f06364837f --- /dev/null +++ b/tests/test_litellm_rust/ocr/conftest.py @@ -0,0 +1,12 @@ +from typing import Final + +import pytest + +from tests.test_litellm_rust.ocr.parity_todo import TODO + + +@pytest.fixture(autouse=True) +def parity_todo(request: pytest.FixtureRequest) -> None: + reason: Final = TODO.get(request.node.name) + if reason is not None: + request.applymarker(pytest.mark.xfail(strict=True, reason=reason)) diff --git a/tests/test_litellm_rust/ocr/parity_todo.py b/tests/test_litellm_rust/ocr/parity_todo.py new file mode 100644 index 00000000000..7f3e9fe34ca --- /dev/null +++ b/tests/test_litellm_rust/ocr/parity_todo.py @@ -0,0 +1,229 @@ +from types import MappingProxyType +from typing import Final + +TODO: Final = MappingProxyType( + { + "test_transport_parity[azure-cohere-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[azure-cohere-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[azure-cohere-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[azure-cohere-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[azure-cohere-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[azure-cohere-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[azure-cohere-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[azure-cohere-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[azure-cohere-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[azure-document-intelligence-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[azure-document-intelligence-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[azure-document-intelligence-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[azure-document-intelligence-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[azure-document-intelligence-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[azure-document-intelligence-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[azure-document-intelligence-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[azure-document-intelligence-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[azure-document-intelligence-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[azure-document-intelligence-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[azure-mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[azure-mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[azure-mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[azure-mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[azure-mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[azure-mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[azure-mistral-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[azure-mistral-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[azure-mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[azure-mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[cohere-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[cohere-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[cohere-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[cohere-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[cohere-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[cohere-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[cohere-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[cohere-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[cohere-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[mistral-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[mistral-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[reducto-legacy-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[reducto-legacy-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[reducto-legacy-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[reducto-legacy-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[reducto-legacy-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[reducto-legacy-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[reducto-legacy-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[reducto-legacy-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[reducto-legacy-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[reducto-v3-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[reducto-v3-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[reducto-v3-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[reducto-v3-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[reducto-v3-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[reducto-v3-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[reducto-v3-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[reducto-v3-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[reducto-v3-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[vertex-deepseek-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[vertex-deepseek-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[vertex-deepseek-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[vertex-deepseek-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[vertex-deepseek-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[vertex-deepseek-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[vertex-deepseek-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[vertex-deepseek-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[vertex-deepseek-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[vertex-deepseek-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[vertex-mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[vertex-mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider", + "test_transport_parity[vertex-mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[vertex-mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation", + "test_transport_parity[vertex-mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[vertex-mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message", + "test_transport_parity[vertex-mistral-oversized-async]": "4.2: Python has no response size limit", + "test_transport_parity[vertex-mistral-oversized-sync]": "4.2: Python has no response size limit", + "test_transport_parity[vertex-mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_transport_parity[vertex-mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's", + "test_validation_parity[azure-cohere-pdf-async]": "4.4: Cohere image-only text differs", + "test_validation_parity[azure-cohere-pdf-sync]": "4.4: Cohere image-only text differs", + "test_validation_parity[bad-req-format-azure-cohere-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-azure-cohere-sync]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-azure-document-intelligence-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-azure-document-intelligence-sync]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-azure-mistral-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-azure-mistral-sync]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-cohere-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-cohere-sync]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-mistral-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-mistral-sync]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-reducto-legacy-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-reducto-legacy-sync]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-reducto-v3-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-reducto-v3-sync]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-vertex-deepseek-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-vertex-deepseek-sync]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-vertex-mistral-async]": "4.4: req_format text differs", + "test_validation_parity[bad-req-format-vertex-mistral-sync]": "4.4: req_format text differs", + "test_validation_parity[cohere-pdf-async]": "4.4: Cohere image-only text differs", + "test_validation_parity[cohere-pdf-sync]": "4.4: Cohere image-only text differs", + "test_validation_parity[empty-file-azure-cohere-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-azure-cohere-sync]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-azure-document-intelligence-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-azure-document-intelligence-sync]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-azure-mistral-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-azure-mistral-sync]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-cohere-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-cohere-sync]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-mistral-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-mistral-sync]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-reducto-legacy-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-reducto-legacy-sync]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-reducto-v3-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-reducto-v3-sync]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-vertex-deepseek-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-vertex-deepseek-sync]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-vertex-mistral-async]": "4.4: local ValueError classification", + "test_validation_parity[empty-file-vertex-mistral-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-data-uri-azure-cohere-async]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-azure-cohere-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-azure-document-intelligence-async]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-azure-document-intelligence-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-azure-mistral-async]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-azure-mistral-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-cohere-async]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-cohere-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-mistral-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-reducto-legacy-async]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-reducto-legacy-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-reducto-v3-async]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-reducto-v3-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-vertex-deepseek-async]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-vertex-deepseek-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-vertex-mistral-async]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-data-uri-vertex-mistral-sync]": "4.4: Rust rejects data URIs Python forwards", + "test_validation_parity[invalid-mime-type-azure-cohere-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-azure-cohere-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-azure-document-intelligence-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-azure-document-intelligence-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-azure-mistral-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-azure-mistral-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-cohere-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-cohere-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-mistral-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-mistral-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-reducto-legacy-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-reducto-legacy-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-reducto-v3-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-reducto-v3-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-vertex-deepseek-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-vertex-deepseek-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-vertex-mistral-async]": "4.4: local ValueError classification", + "test_validation_parity[invalid-mime-type-vertex-mistral-sync]": "4.4: local ValueError classification", + "test_validation_parity[invalid-pages-async]": "4.4: pages validation text differs", + "test_validation_parity[invalid-pages-sync]": "4.4: pages validation text differs", + "test_validation_parity[invalid-provider-async]": "4.4: invalid provider text differs", + "test_validation_parity[invalid-provider-sync]": "4.4: invalid provider text differs", + "test_validation_parity[missing-azure-ai-credentials-async]": "4.4: credential text differs", + "test_validation_parity[missing-azure-ai-credentials-sync]": "4.4: credential text differs", + "test_validation_parity[missing-document-intelligence-credentials-async]": "4.4: credential text differs", + "test_validation_parity[missing-document-intelligence-credentials-sync]": "4.4: credential text differs", + "test_validation_parity[missing-document-url-azure-cohere-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-azure-cohere-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-azure-document-intelligence-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-azure-document-intelligence-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-azure-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-azure-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-cohere-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-cohere-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-reducto-legacy-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-reducto-legacy-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-reducto-v3-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-reducto-v3-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-vertex-deepseek-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-vertex-deepseek-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-vertex-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-document-url-vertex-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution", + "test_validation_parity[missing-file-cohere-async]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-cohere-sync]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-reducto-legacy-async]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-reducto-legacy-sync]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-reducto-v3-async]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-reducto-v3-sync]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-vertex-deepseek-async]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-vertex-deepseek-sync]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-vertex-mistral-async]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-file-vertex-mistral-sync]": "4.4: local FileNotFoundError classification", + "test_validation_parity[missing-reducto-key-async]": "4.4: credential error classification", + "test_validation_parity[missing-reducto-key-sync]": "4.4: credential error classification", + "test_validation_parity[reducto-legacy-plain-url-async]": "4.4: Reducto source text differs", + "test_validation_parity[reducto-legacy-plain-url-sync]": "4.4: Reducto source text differs", + "test_validation_parity[reducto-plain-url-async]": "4.4: Reducto source text differs", + "test_validation_parity[reducto-plain-url-sync]": "4.4: Reducto source text differs", + "test_validation_parity[unreadable-file-azure-cohere-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-azure-cohere-sync]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-azure-document-intelligence-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-azure-document-intelligence-sync]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-azure-mistral-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-azure-mistral-sync]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-cohere-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-cohere-sync]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-mistral-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-mistral-sync]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-reducto-legacy-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-reducto-legacy-sync]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-reducto-v3-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-reducto-v3-sync]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-vertex-deepseek-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-vertex-deepseek-sync]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-vertex-mistral-async]": "4.4: local OSError classification", + "test_validation_parity[unreadable-file-vertex-mistral-sync]": "4.4: local OSError classification", + "test_router_and_proxy_parity[missing-document-url]": "4.4: Python rejects an empty document URL before provider resolution", + } +) diff --git a/tests/test_litellm_rust/ocr/test_callbacks.py b/tests/test_litellm_rust/ocr/test_callbacks.py index ac4a1a11a80..44aed91d486 100644 --- a/tests/test_litellm_rust/ocr/test_callbacks.py +++ b/tests/test_litellm_rust/ocr/test_callbacks.py @@ -44,6 +44,24 @@ async def call_native_aocr_with_callbacks(server: RecordingServer, callbacks: li return await call_native_aocr(server, callbacks=callbacks, **kwargs) +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_ocr_contract_post_call_event_carries_the_api_key_and_only_the_request_body( + ocr_server: RecordingServer, ocr_backend: bool, asynchronous: bool +) -> None: + observations: Final = [] + + class Observe(CustomLogger): + def log_post_api_call(self, kwargs, response_obj, start_time, end_time): + observations.append((kwargs["api_key"], copy.deepcopy(kwargs["additional_args"]))) + + await call_native(ocr_server, asynchronous, callbacks=[Observe()]) + + assert observations == [ + ("test-key", {"complete_input_dict": {"model": "mistral-ocr-latest", "document": OCR_DOCUMENT}}) + ] + + def test_native_ocr_pre_call_callback_receives_transformed_provider_request(ocr_server: RecordingServer) -> None: observations: Final = [] diff --git a/tests/test_litellm_rust/ocr/test_parity.py b/tests/test_litellm_rust/ocr/test_parity.py new file mode 100644 index 00000000000..74cafff37b3 --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_parity.py @@ -0,0 +1,357 @@ +import re +import socket +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from pathlib import Path +from types import MappingProxyType +from typing import Final + +import pytest + +import litellm +from litellm.llms.base_llm.ocr.transformation import OCRResponse +from tests.test_litellm_rust.support.parity import Divergence, Masked, Outcome, assert_parity, run_on +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import OCR_DOCUMENT + +pytestmark = pytest.mark.requires_rust_extension + +IMAGE_DOCUMENT: Final = {"type": "image_url", "image_url": "data:image/png;base64,YWJj"} +REDUCTO_DOCUMENT: Final = {"type": "document_url", "document_url": "reducto://ready.pdf"} +VERTEX: Final = {"vertex_project": "test-project", "vertex_location": "us-central1"} + + +@dataclass(frozen=True, slots=True) +class Provider: + model: str + document: Mapping[str, object] + arguments: Mapping[str, object] + + +PROVIDERS: Final = { + "mistral": Provider("mistral/mistral-ocr-latest", OCR_DOCUMENT, {}), + "azure-mistral": Provider("azure_ai/mistral-ocr-latest", OCR_DOCUMENT, {}), + "azure-document-intelligence": Provider("azure_ai/doc-intelligence/prebuilt-layout", OCR_DOCUMENT, {}), + "azure-cohere": Provider("azure_ai/Cohere-parse-v5.0", IMAGE_DOCUMENT, {}), + "cohere": Provider("cohere/parse-v5.0", IMAGE_DOCUMENT, {}), + "vertex-mistral": Provider("vertex_ai/mistral-ocr-latest", OCR_DOCUMENT, VERTEX), + "vertex-deepseek": Provider("vertex_ai/deepseek-ocr", OCR_DOCUMENT, VERTEX), + "reducto-v3": Provider("reducto/parse-v3", REDUCTO_DOCUMENT, {}), + "reducto-legacy": Provider("reducto/parse-legacy", REDUCTO_DOCUMENT, {}), +} + +STATUSES: Final = (400, 401, 403, 404, 408, 409, 413, 422, 429, 498, 500, 502, 503, 504, 529) + +BODIES: Final = { + "json-object": {"message": "rejected"}, + "json-string": "rejected", + "empty-object": {}, + "rate-limit-text": {"message": "rate limit reached for requests"}, + "context-window-text": {"message": "This model's maximum context length is 10 tokens"}, + "content-policy-text": {"message": "content_policy_violation"}, + "invalid-request-text": {"error": {"type": "invalid_request_error", "message": "bad field"}}, + "model-not-found-text": {"error": {"type": "invalid_request_error", "code": "model_not_found"}}, + "timeout-text": {"message": "Request timed out"}, + "vertex-quota-text": {"message": "429 Quota exceeded for aiplatform"}, + "vertex-api-key-text": {"message": "API key not valid. Please pass a valid API key."}, + "cohere-api-token-text": {"message": "invalid api token"}, + "cohere-internal-text": {"message": "Internal Server Error"}, +} + +REPRESENTATIVE_STATUSES: Final = (400, 429, 500) + +NO_RETRIES: Final = MappingProxyType({"num_retries": 0}) +CALL_OPTIONS: Final = { + "no-retries": NO_RETRIES, + "retries-and-timeout": MappingProxyType({"num_retries": 0, "timeout": 7.5}), + "defaults": MappingProxyType({}), +} + +VERTEX_UPSTREAM_CONTENT: Final = tuple( + Divergence(field, "Rust keeps the upstream body and headers that _map_vertex_exception replaces with a stub") + for field in ("response_text", "response_headers") +) + + +def upstream_divergences( + asynchronous: bool, provider: Provider, options: Mapping[str, object], python: Outcome +) -> tuple[Divergence, ...]: + sync: Final = tuple( + Divergence(field, "Rust sets the request retry count and timeout on sync failures; Python only on async") + for field in ("num_retries", "timeout") + if not asynchronous and options.get(field) is not None + ) + vertex: Final = provider.model.startswith("vertex_ai/") and python.response_text == "" + return sync + (VERTEX_UPSTREAM_CONTENT if vertex else ()) + + +def arguments(server: RecordingServer, provider: Provider, **extra: object) -> dict[str, object]: + return { + "model": provider.model, + "document": dict(provider.document), + "api_key": "test-key", + "api_base": server.base_url, + **provider.arguments, + **extra, + } + + +async def call( + server: RecordingServer, asynchronous: bool, provider: Provider, options: Mapping[str, object] +) -> OCRResponse: + if asynchronous: + return await litellm.aocr(**arguments(server, provider, **options)) + return litellm.ocr(**arguments(server, provider, **options)) + + +async def compare_upstream( + server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + provider: Provider, + response: ResponseSpec, + asynchronous: bool, + options: Mapping[str, object] = NO_RETRIES, + masks: tuple[Masked, ...] = (), +) -> None: + server.expected_requests = None + server.enqueue(response) + python: Final = await run_on(False, monkeypatch, lambda: call(server, asynchronous, provider, options)) + server.enqueue(response) + rust: Final = await run_on(True, monkeypatch, lambda: call(server, asynchronous, provider, options)) + assert_parity(python, rust, upstream_divergences(asynchronous, provider, options, python), masks) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("status", STATUSES) +@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys()) +async def test_upstream_status_parity( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + provider: Provider, + status: int, + asynchronous: bool, +) -> None: + await compare_upstream( + recording_server, + monkeypatch, + provider, + ResponseSpec(body=BODIES["json-object"], status=status, headers={"retry-after": "7"}), + asynchronous, + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("status", REPRESENTATIVE_STATUSES) +@pytest.mark.parametrize("body", BODIES.values(), ids=BODIES.keys()) +@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys()) +async def test_upstream_body_parity( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + provider: Provider, + body: object, + status: int, + asynchronous: bool, +) -> None: + await compare_upstream( + recording_server, monkeypatch, provider, ResponseSpec(body=body, status=status), asynchronous + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("options", CALL_OPTIONS.values(), ids=CALL_OPTIONS.keys()) +@pytest.mark.parametrize("status", (401, 500)) +@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys()) +async def test_call_option_parity( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + provider: Provider, + status: int, + options: Mapping[str, object], + asynchronous: bool, +) -> None: + await compare_upstream( + recording_server, + monkeypatch, + provider, + ResponseSpec(body=BODIES["json-object"], status=status), + asynchronous, + options, + ) + + +def closed_port_base_url() -> str: + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + host, port = probe.getsockname() + return f"http://{host}:{port}" + + +@dataclass(frozen=True, slots=True) +class Transport: + response: ResponseSpec | None + options: Mapping[str, object] + + +TRANSPORTS: Final = { + "read-timeout": Transport(ResponseSpec(body={}, delay=1.5), {**NO_RETRIES, "timeout": 0.5}), + "connection-refused": Transport(None, NO_RETRIES), + "not-json": Transport(ResponseSpec(body=None, raw=b"bad gateway"), NO_RETRIES), + "missing-fields": Transport(ResponseSpec(body={"unexpected": True}), NO_RETRIES), + "oversized": Transport(ResponseSpec(body={"padding": "x" * 64}), {**NO_RETRIES, "max_response_bytes": 16}), +} + + +def replaced(pattern: re.Pattern[str], placeholder: str) -> Callable[[object], object]: + return lambda value: pattern.sub(placeholder, value) if isinstance(value, str) else value + + +ELAPSED: Final = tuple( + Masked( + field, + "both handlers report how long the timed-out call ran", + replaced(re.compile(r"time taken=[0-9.]+ seconds"), "time taken= seconds"), + ) + for field in ("text", "message") +) + +REFUSED_CONNECTION: Final = re.compile(r"\[Errno \d+\] Connection refused|error sending request") + + +def refused_connection_divergences(provider: Provider) -> tuple[Divergence, ...]: + reason: Final = "httpx and reqwest word a refused connection differently" + text: Final = tuple( + Divergence(field, reason, replaced(REFUSED_CONNECTION, "")) for field in ("text", "message") + ) + vertex: Final = (Divergence("response_text", reason),) if provider.model.startswith("vertex_ai/") else () + return text + vertex + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("transport", TRANSPORTS.values(), ids=TRANSPORTS.keys()) +@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys()) +async def test_transport_parity( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + provider: Provider, + transport: Transport, + asynchronous: bool, +) -> None: + if transport.response is not None: + await compare_upstream( + recording_server, monkeypatch, provider, transport.response, asynchronous, transport.options, ELAPSED + ) + return + recording_server.expected_requests = 0 + unreachable: Final = {**transport.options, "api_base": closed_port_base_url()} + python: Final = await run_on( + False, monkeypatch, lambda: call(recording_server, asynchronous, provider, unreachable) + ) + rust: Final = await run_on(True, monkeypatch, lambda: call(recording_server, asynchronous, provider, unreachable)) + assert_parity( + python, + rust, + upstream_divergences(asynchronous, provider, unreachable, python) + refused_connection_divergences(provider), + ) + + +@dataclass(frozen=True, slots=True) +class Validation: + document: Callable[[Path], Mapping[str, object]] | None = None + options: Mapping[str, object] = NO_RETRIES + api_key: str | None = "test-key" + + +def file_document(file: object, **extra: object) -> Mapping[str, object]: + return {"type": "file", "file": file, **extra} + + +def unreadable_file(directory: Path) -> Mapping[str, object]: + path: Final = directory / "unreadable.pdf" + if not path.exists(): + path.write_bytes(b"%PDF-1.4") + path.chmod(0) + return file_document(path) + + +class RaisingReader: + def read(self) -> bytes: + raise ValueError("reader exploded") + + +def raising_token_provider() -> str: + raise ValueError("token unavailable") + + +def url_document(url: str) -> Callable[[Path], Mapping[str, object]]: + return lambda _: {"type": "document_url", "document_url": url} + + +GENERIC_VALIDATIONS: Final = { + "bad-req-format": Validation(options={**NO_RETRIES, "req_format": "it's-bogus"}), + "empty-file": Validation(lambda _: file_document(b"")), + "missing-file": Validation(lambda directory: file_document(directory / "missing.pdf")), + "unreadable-file": Validation(unreadable_file), + "missing-document-url": Validation(url_document("")), + "invalid-mime-type": Validation(lambda _: file_document(b"abc", mime_type="not a mime type")), + "invalid-data-uri": Validation(url_document("data:application/pdf;base64")), +} + +SPECIFIC_VALIDATIONS: Final = { + "cohere-pdf": ("cohere", Validation(url_document(str(OCR_DOCUMENT["document_url"])))), + "azure-cohere-pdf": ("azure-cohere", Validation(url_document(str(OCR_DOCUMENT["document_url"])))), + "reducto-plain-url": ("reducto-v3", Validation(url_document("https://example.com/scan.pdf"))), + "reducto-legacy-plain-url": ("reducto-legacy", Validation(url_document("https://example.com/scan.pdf"))), + "invalid-pages": ("azure-document-intelligence", Validation(options={**NO_RETRIES, "pages": [-1]})), + "missing-azure-ai-credentials": ("azure-mistral", Validation(api_key=None)), + "missing-document-intelligence-credentials": ("azure-document-intelligence", Validation(api_key=None)), + "missing-reducto-key": ("reducto-v3", Validation(api_key=None)), + "raising-file-reader": ("mistral", Validation(lambda _: file_document(RaisingReader()))), + "raising-token-provider": ( + "azure-mistral", + Validation(options={**NO_RETRIES, "azure_ad_token_provider": raising_token_provider}, api_key=None), + ), +} + +VALIDATION_ROWS: Final = { + **{ + f"{name}-{provider}": (PROVIDERS[provider], validation) + for name, validation in GENERIC_VALIDATIONS.items() + for provider in PROVIDERS + }, + **{name: (PROVIDERS[provider], validation) for name, (provider, validation) in SPECIFIC_VALIDATIONS.items()}, + "invalid-provider": (Provider("not_a_provider/ocr-model", OCR_DOCUMENT, {}), Validation()), +} + +CREDENTIAL_VARIABLES: Final = ("AZURE_DOCUMENT_INTELLIGENCE_API_KEY", "REDUCTO_API_KEY", "MISTRAL_API_KEY") + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("row", VALIDATION_ROWS.values(), ids=VALIDATION_ROWS.keys()) +async def test_validation_parity( + recording_server: RecordingServer, + monkeypatch: pytest.MonkeyPatch, + isolated_azure_auth: None, + tmp_path: Path, + row: tuple[Provider, Validation], + asynchronous: bool, +) -> None: + provider, validation = row + for variable in CREDENTIAL_VARIABLES: + monkeypatch.delenv(variable, raising=False) + recording_server.expected_requests = None + + def invoke() -> Awaitable[OCRResponse]: + document: Final = provider.document if validation.document is None else validation.document(tmp_path) + options: Final = {**validation.options, "document": dict(document), "api_key": validation.api_key} + return call(recording_server, asynchronous, provider, options) + + python: Final = await run_on(False, monkeypatch, invoke) + python_requests: Final = len(recording_server.requests) + rust: Final = await run_on(True, monkeypatch, invoke) + assert len(recording_server.requests) - python_requests == python_requests + assert_parity(python, rust, upstream_divergences(asynchronous, provider, validation.options, python)) diff --git a/tests/test_litellm_rust/ocr/test_requests.py b/tests/test_litellm_rust/ocr/test_requests.py index 5e9d2c78808..4c4e595dfb4 100644 --- a/tests/test_litellm_rust/ocr/test_requests.py +++ b/tests/test_litellm_rust/ocr/test_requests.py @@ -24,13 +24,6 @@ from tests.test_litellm_rust.support.requests import ( pytestmark = pytest.mark.requires_rust_extension -@pytest.fixture(params=[False, True], ids=["python", "rust"]) -def ocr_backend(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> bool: - enabled: Final = bool(request.param) - monkeypatch.setenv("LITELLM_RUST", "1" if enabled else "0") - return enabled - - @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) async def test_ocr_contract_upstream_status( diff --git a/tests/test_litellm_rust/ocr/test_router_and_proxy.py b/tests/test_litellm_rust/ocr/test_router_and_proxy.py new file mode 100644 index 00000000000..e99d5f438d4 --- /dev/null +++ b/tests/test_litellm_rust/ocr/test_router_and_proxy.py @@ -0,0 +1,98 @@ +from collections.abc import Mapping +from dataclasses import dataclass +from types import MappingProxyType +from typing import Final + +import pytest +from fastapi.testclient import TestClient + +from litellm import Router +from litellm.proxy import proxy_server +from tests.test_litellm_rust.support.parity import assert_parity +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import OCR_DOCUMENT, OCR_MODEL, OCR_RESPONSE + +pytestmark = pytest.mark.requires_rust_extension + +MODEL_GROUP: Final = "ocr" +ROUTER_RETRIES: Final = 1 + + +@dataclass(frozen=True, slots=True) +class Row: + upstream: ResponseSpec + document: Mapping[str, object] = MappingProxyType(dict(OCR_DOCUMENT)) + deployment: Mapping[str, object] = MappingProxyType({}) + + +@dataclass(frozen=True, slots=True) +class ProxyOutcome: + status: int + error_type: object + error_code: object + upstream_requests: int + cooled_down: int + timeout_header: bool + markdown: object + + +ROWS: Final = { + "success": Row(ResponseSpec(body=OCR_RESPONSE)), + "unauthorized": Row(ResponseSpec(body={"message": "rejected"}, status=401)), + "not-found": Row(ResponseSpec(body={"message": "no such model"}, status=404)), + "rate-limited": Row(ResponseSpec(body={"message": "slow down"}, status=429, headers={"retry-after": "0"})), + "provider-failure": Row(ResponseSpec(body={"message": "provider unavailable"}, status=500)), + "read-timeout": Row(ResponseSpec(body=OCR_RESPONSE, delay=1.5), deployment=MappingProxyType({"timeout": 0.5})), + "not-json": Row(ResponseSpec(body=None, raw=b"bad gateway")), + "missing-document-url": Row( + ResponseSpec(body=OCR_RESPONSE), document=MappingProxyType({"type": "document_url", "document_url": ""}) + ), +} + + +def observe(server: RecordingServer, row: Row, monkeypatch: pytest.MonkeyPatch, backend: bool) -> ProxyOutcome: + monkeypatch.setenv("LITELLM_RUST", "1" if backend else "0") + server.default_response = row.upstream + before: Final = len(server.requests) + router: Final = Router( + model_list=[ + { + "model_name": MODEL_GROUP, + "litellm_params": { + "model": OCR_MODEL, + "api_key": "test-key", + "api_base": server.base_url, + **row.deployment, + }, + } + ], + num_retries=ROUTER_RETRIES, + retry_after=0, + ) + monkeypatch.setattr(proxy_server, "llm_router", router) + monkeypatch.setattr(proxy_server, "master_key", None) + monkeypatch.setattr(proxy_server, "general_settings", {}) + with TestClient(proxy_server.app, raise_server_exceptions=False) as client: + response: Final = client.post("/v1/ocr", json={"model": MODEL_GROUP, "document": dict(row.document)}) + body: Final = response.json() + error: Final = body.get("error") if isinstance(body, dict) else None + pages: Final = body.get("pages") if isinstance(body, dict) else None + if response.status_code == 200: + assert response.headers.get("x-litellm-rust") == ("true" if backend else None) + return ProxyOutcome( + status=response.status_code, + error_type=None if not isinstance(error, dict) else error.get("type"), + error_code=None if not isinstance(error, dict) else error.get("code"), + upstream_requests=len(server.requests) - before, + cooled_down=len(router.cooldown_cache.get_active_cooldowns(router.get_model_ids(), parent_otel_span=None)), + timeout_header="x-litellm-timeout" in response.headers, + markdown=pages[0].get("markdown") if isinstance(pages, list) and pages else None, + ) + + +@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys()) +def test_router_and_proxy_parity(recording_server: RecordingServer, monkeypatch: pytest.MonkeyPatch, row: Row) -> None: + recording_server.expected_requests = None + python: Final = observe(recording_server, row, monkeypatch, backend=False) + rust: Final = observe(recording_server, row, monkeypatch, backend=True) + assert_parity(python, rust) diff --git a/tests/test_litellm_rust/support/parity.py b/tests/test_litellm_rust/support/parity.py new file mode 100644 index 00000000000..187ae61cf0b --- /dev/null +++ b/tests/test_litellm_rust/support/parity.py @@ -0,0 +1,142 @@ +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass, fields, replace +from types import MappingProxyType +from typing import TYPE_CHECKING, Final, TypeVar + +import httpx +import pytest + +from litellm.llms.base_llm.ocr.transformation import OCRResponse + +if TYPE_CHECKING: + from _typeshed import DataclassInstance + +VOLATILE_HEADERS: Final = frozenset({"date", "server"}) + + +@dataclass(frozen=True, slots=True) +class Outcome: + kind: str + error_class: str | None = None + text: str | None = None + message: str | None = None + status_code: object = None + llm_provider: object = None + model: object = None + litellm_debug_info: object = None + num_retries: object = None + max_retries: object = None + timeout: object = None + response_status: int | None = None + response_text: str | None = None + response_headers: Mapping[str, str] | None = None + litellm_response_headers: Mapping[str, str] | None = None + result: object = None + + +def _headers(value: object) -> Mapping[str, str] | None: + if not isinstance(value, httpx.Headers | Mapping): + return None + return MappingProxyType( + {str(name).lower(): str(item) for name, item in value.items() if str(name).lower() not in VOLATILE_HEADERS} + ) + + +def _response(error: BaseException) -> httpx.Response | None: + try: + response: Final = getattr(error, "response", None) + except RuntimeError: + return None + return response if isinstance(response, httpx.Response) else None + + +def failure_outcome(error: BaseException) -> Outcome: + response: Final = _response(error) + return Outcome( + kind="error", + error_class=f"{type(error).__module__}.{type(error).__qualname__}", + text=str(error), + message=getattr(error, "message", None), + status_code=getattr(error, "status_code", None), + llm_provider=getattr(error, "llm_provider", None), + model=getattr(error, "model", None), + litellm_debug_info=getattr(error, "litellm_debug_info", None), + num_retries=getattr(error, "num_retries", None), + max_retries=getattr(error, "max_retries", None), + timeout=getattr(error, "timeout", None), + response_status=None if response is None else response.status_code, + response_text=None if response is None else response.text, + response_headers=None if response is None else _headers(response.headers), + litellm_response_headers=_headers(getattr(error, "litellm_response_headers", None)), + ) + + +def success_outcome(response: OCRResponse) -> Outcome: + return Outcome(kind="ok", result=response.model_dump()) + + +@dataclass(frozen=True, slots=True) +class Divergence: + """A field that differs by design. With `normalize`, both sides are compared after it + instead of the field being dropped.""" + + field: str + reason: str + normalize: Callable[[object], object] | None = None + + +@dataclass(frozen=True, slots=True) +class Masked: + """A field that holds a run-dependent value on both sides, compared after `mask`.""" + + field: str + reason: str + mask: Callable[[object], object] + + +async def observe(call: Callable[[], Awaitable[OCRResponse]]) -> Outcome: + try: + return success_outcome(await call()) + except Exception as error: + return failure_outcome(error) + + +async def run_on(backend: bool, monkeypatch: pytest.MonkeyPatch, call: Callable[[], Awaitable[OCRResponse]]) -> Outcome: + monkeypatch.setenv("LITELLM_RUST", "1" if backend else "0") + return await observe(call) + + +ObservedT = TypeVar("ObservedT", bound="DataclassInstance") + + +def without(outcome: ObservedT, divergences: tuple[Divergence, ...], masks: tuple[Masked, ...] = ()) -> ObservedT: + compared: Final = { + **{ + divergence.field: None + if divergence.normalize is None + else divergence.normalize(getattr(outcome, divergence.field)) + for divergence in divergences + }, + **{masked.field: masked.mask(getattr(outcome, masked.field)) for masked in masks}, + } + return replace(outcome, **compared) + + +def assert_parity( + python: ObservedT, rust: ObservedT, divergences: tuple[Divergence, ...] = (), masks: tuple[Masked, ...] = () +) -> None: + known: Final = frozenset(divergence.field for divergence in divergences) | {masked.field for masked in masks} + unknown: Final = known - {field.name for field in fields(python)} + assert not unknown, f"divergence registry names unknown fields: {sorted(unknown)}" + for divergence in divergences: + assert getattr(python, divergence.field) != getattr(rust, divergence.field), ( + f"registered divergence on {divergence.field!r} no longer diverges; remove it: {divergence.reason}" + ) + expected: Final = without(python, divergences, masks) + actual: Final = without(rust, divergences, masks) + differences: Final = { + field.name: {"python": getattr(expected, field.name), "rust": getattr(actual, field.name)} + for field in fields(python) + if getattr(expected, field.name) != getattr(actual, field.name) + } + assert not differences, differences diff --git a/tests/test_litellm_rust/support/recording_server.py b/tests/test_litellm_rust/support/recording_server.py index 3eea47751d3..d084c38b92e 100644 --- a/tests/test_litellm_rust/support/recording_server.py +++ b/tests/test_litellm_rust/support/recording_server.py @@ -26,8 +26,11 @@ class ResponseSpec: headers: dict[str, str] = field(default_factory=dict) delay: float = 0 events: tuple[tuple[str, object], ...] = () + raw: bytes | None = None def payloads(self) -> tuple[bytes, ...]: + if self.raw is not None: + return (self.raw,) if not self.events: return (json.dumps(self.body).encode(),) return tuple(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() for event, data in self.events)