test(rust): pin Vertex OCR hosts for global and multi-region locations

The Python path picks the host with get_vertex_base_url: global uses
aiplatform.googleapis.com and us/eu use the .rep. hosts. The Rust config
always builds {location}-aiplatform.googleapis.com, so these cases fail
until it matches
This commit is contained in:
Yujong Lee 2026-09-18 17:00:02 -07:00
parent ff7dc86947
commit c20ae5fffb
10 changed files with 882 additions and 11 deletions

View file

@ -204,15 +204,27 @@ fn validate_location(location: &str) -> Result<(), Error> {
#[cfg(test)]
mod tests {
use rstest::rstest;
use super::VertexAiOcrConfig;
#[test]
fn endpoint_uses_location_project_and_model() {
/// The hosts `get_vertex_base_url` in `litellm/llms/vertex_ai/common_utils.py` picks.
#[rstest]
#[case::regional("europe-west4", "https://europe-west4-aiplatform.googleapis.com")]
#[case::global("global", "https://aiplatform.googleapis.com")]
#[case::multi_region_us("us", "https://aiplatform.us.rep.googleapis.com")]
#[case::multi_region_eu("eu", "https://aiplatform.eu.rep.googleapis.com")]
fn endpoint_uses_the_location_host_project_and_model(
#[case] location: &str,
#[case] host: &str,
) {
assert_eq!(
VertexAiOcrConfig
.build_ocr_url(None, "proj-1", "europe-west4", "mistral-ocr-maas")
.build_ocr_url(None, "proj-1", location, "mistral-ocr-maas")
.unwrap(),
"https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
format!(
"{host}/v1/projects/proj-1/locations/{location}/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
)
);
}

View file

@ -41,6 +41,13 @@ async def isolate_ocr_test_state() -> AsyncIterator[None]:
await GLOBAL_LOGGING_WORKER.stop()
@pytest.fixture(params=[False, True], ids=["python", "rust"])
def ocr_backend(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> bool:
enabled: Final = bool(request.param)
monkeypatch.setenv("LITELLM_RUST", "1" if enabled else "0")
return enabled
@pytest.fixture
def recording_server() -> Generator[RecordingServer]:
with recording_service() as server:

View file

@ -0,0 +1,12 @@
from typing import Final
import pytest
from tests.test_litellm_rust.ocr.parity_todo import TODO
@pytest.fixture(autouse=True)
def parity_todo(request: pytest.FixtureRequest) -> None:
reason: Final = TODO.get(request.node.name)
if reason is not None:
request.applymarker(pytest.mark.xfail(strict=True, reason=reason))

View file

@ -0,0 +1,229 @@
from types import MappingProxyType
from typing import Final
TODO: Final = MappingProxyType(
{
"test_transport_parity[azure-cohere-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[azure-cohere-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[azure-cohere-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[azure-cohere-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[azure-cohere-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[azure-cohere-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[azure-cohere-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[azure-cohere-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[azure-cohere-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[azure-document-intelligence-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[azure-document-intelligence-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[azure-document-intelligence-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[azure-document-intelligence-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[azure-document-intelligence-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[azure-document-intelligence-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[azure-document-intelligence-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[azure-document-intelligence-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[azure-document-intelligence-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[azure-document-intelligence-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[azure-mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[azure-mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[azure-mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[azure-mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[azure-mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[azure-mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[azure-mistral-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[azure-mistral-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[azure-mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[azure-mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[cohere-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[cohere-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[cohere-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[cohere-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[cohere-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[cohere-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[cohere-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[cohere-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[cohere-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[mistral-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[mistral-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[reducto-legacy-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[reducto-legacy-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[reducto-legacy-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[reducto-legacy-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[reducto-legacy-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[reducto-legacy-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[reducto-legacy-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[reducto-legacy-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[reducto-legacy-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[reducto-v3-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[reducto-v3-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[reducto-v3-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[reducto-v3-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[reducto-v3-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[reducto-v3-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[reducto-v3-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[reducto-v3-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[reducto-v3-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[vertex-deepseek-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[vertex-deepseek-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[vertex-deepseek-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[vertex-deepseek-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[vertex-deepseek-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[vertex-deepseek-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[vertex-deepseek-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[vertex-deepseek-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[vertex-deepseek-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[vertex-deepseek-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[vertex-mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[vertex-mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
"test_transport_parity[vertex-mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[vertex-mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
"test_transport_parity[vertex-mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[vertex-mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
"test_transport_parity[vertex-mistral-oversized-async]": "4.2: Python has no response size limit",
"test_transport_parity[vertex-mistral-oversized-sync]": "4.2: Python has no response size limit",
"test_transport_parity[vertex-mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_transport_parity[vertex-mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
"test_validation_parity[azure-cohere-pdf-async]": "4.4: Cohere image-only text differs",
"test_validation_parity[azure-cohere-pdf-sync]": "4.4: Cohere image-only text differs",
"test_validation_parity[bad-req-format-azure-cohere-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-azure-cohere-sync]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-azure-document-intelligence-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-azure-document-intelligence-sync]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-azure-mistral-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-azure-mistral-sync]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-cohere-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-cohere-sync]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-mistral-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-mistral-sync]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-reducto-legacy-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-reducto-legacy-sync]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-reducto-v3-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-reducto-v3-sync]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-vertex-deepseek-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-vertex-deepseek-sync]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-vertex-mistral-async]": "4.4: req_format text differs",
"test_validation_parity[bad-req-format-vertex-mistral-sync]": "4.4: req_format text differs",
"test_validation_parity[cohere-pdf-async]": "4.4: Cohere image-only text differs",
"test_validation_parity[cohere-pdf-sync]": "4.4: Cohere image-only text differs",
"test_validation_parity[empty-file-azure-cohere-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-azure-cohere-sync]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-azure-document-intelligence-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-azure-document-intelligence-sync]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-azure-mistral-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-azure-mistral-sync]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-cohere-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-cohere-sync]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-mistral-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-mistral-sync]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-reducto-legacy-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-reducto-legacy-sync]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-reducto-v3-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-reducto-v3-sync]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-vertex-deepseek-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-vertex-deepseek-sync]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-vertex-mistral-async]": "4.4: local ValueError classification",
"test_validation_parity[empty-file-vertex-mistral-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-data-uri-azure-cohere-async]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-azure-cohere-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-azure-document-intelligence-async]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-azure-document-intelligence-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-azure-mistral-async]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-azure-mistral-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-cohere-async]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-cohere-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-mistral-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-reducto-legacy-async]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-reducto-legacy-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-reducto-v3-async]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-reducto-v3-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-vertex-deepseek-async]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-vertex-deepseek-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-vertex-mistral-async]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-data-uri-vertex-mistral-sync]": "4.4: Rust rejects data URIs Python forwards",
"test_validation_parity[invalid-mime-type-azure-cohere-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-azure-cohere-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-azure-document-intelligence-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-azure-document-intelligence-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-azure-mistral-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-azure-mistral-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-cohere-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-cohere-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-mistral-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-mistral-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-reducto-legacy-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-reducto-legacy-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-reducto-v3-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-reducto-v3-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-vertex-deepseek-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-vertex-deepseek-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-vertex-mistral-async]": "4.4: local ValueError classification",
"test_validation_parity[invalid-mime-type-vertex-mistral-sync]": "4.4: local ValueError classification",
"test_validation_parity[invalid-pages-async]": "4.4: pages validation text differs",
"test_validation_parity[invalid-pages-sync]": "4.4: pages validation text differs",
"test_validation_parity[invalid-provider-async]": "4.4: invalid provider text differs",
"test_validation_parity[invalid-provider-sync]": "4.4: invalid provider text differs",
"test_validation_parity[missing-azure-ai-credentials-async]": "4.4: credential text differs",
"test_validation_parity[missing-azure-ai-credentials-sync]": "4.4: credential text differs",
"test_validation_parity[missing-document-intelligence-credentials-async]": "4.4: credential text differs",
"test_validation_parity[missing-document-intelligence-credentials-sync]": "4.4: credential text differs",
"test_validation_parity[missing-document-url-azure-cohere-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-azure-cohere-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-azure-document-intelligence-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-azure-document-intelligence-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-azure-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-azure-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-cohere-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-cohere-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-reducto-legacy-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-reducto-legacy-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-reducto-v3-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-reducto-v3-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-vertex-deepseek-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-vertex-deepseek-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-vertex-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-document-url-vertex-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution",
"test_validation_parity[missing-file-cohere-async]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-cohere-sync]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-reducto-legacy-async]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-reducto-legacy-sync]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-reducto-v3-async]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-reducto-v3-sync]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-vertex-deepseek-async]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-vertex-deepseek-sync]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-vertex-mistral-async]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-file-vertex-mistral-sync]": "4.4: local FileNotFoundError classification",
"test_validation_parity[missing-reducto-key-async]": "4.4: credential error classification",
"test_validation_parity[missing-reducto-key-sync]": "4.4: credential error classification",
"test_validation_parity[reducto-legacy-plain-url-async]": "4.4: Reducto source text differs",
"test_validation_parity[reducto-legacy-plain-url-sync]": "4.4: Reducto source text differs",
"test_validation_parity[reducto-plain-url-async]": "4.4: Reducto source text differs",
"test_validation_parity[reducto-plain-url-sync]": "4.4: Reducto source text differs",
"test_validation_parity[unreadable-file-azure-cohere-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-azure-cohere-sync]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-azure-document-intelligence-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-azure-document-intelligence-sync]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-azure-mistral-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-azure-mistral-sync]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-cohere-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-cohere-sync]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-mistral-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-mistral-sync]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-reducto-legacy-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-reducto-legacy-sync]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-reducto-v3-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-reducto-v3-sync]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-vertex-deepseek-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-vertex-deepseek-sync]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-vertex-mistral-async]": "4.4: local OSError classification",
"test_validation_parity[unreadable-file-vertex-mistral-sync]": "4.4: local OSError classification",
"test_router_and_proxy_parity[missing-document-url]": "4.4: Python rejects an empty document URL before provider resolution",
}
)

View file

@ -44,6 +44,24 @@ async def call_native_aocr_with_callbacks(server: RecordingServer, callbacks: li
return await call_native_aocr(server, callbacks=callbacks, **kwargs)
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_ocr_contract_post_call_event_carries_the_api_key_and_only_the_request_body(
ocr_server: RecordingServer, ocr_backend: bool, asynchronous: bool
) -> None:
observations: Final = []
class Observe(CustomLogger):
def log_post_api_call(self, kwargs, response_obj, start_time, end_time):
observations.append((kwargs["api_key"], copy.deepcopy(kwargs["additional_args"])))
await call_native(ocr_server, asynchronous, callbacks=[Observe()])
assert observations == [
("test-key", {"complete_input_dict": {"model": "mistral-ocr-latest", "document": OCR_DOCUMENT}})
]
def test_native_ocr_pre_call_callback_receives_transformed_provider_request(ocr_server: RecordingServer) -> None:
observations: Final = []

View file

@ -0,0 +1,357 @@
import re
import socket
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass
from pathlib import Path
from types import MappingProxyType
from typing import Final
import pytest
import litellm
from litellm.llms.base_llm.ocr.transformation import OCRResponse
from tests.test_litellm_rust.support.parity import Divergence, Masked, Outcome, assert_parity, run_on
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
from tests.test_litellm_rust.support.requests import OCR_DOCUMENT
pytestmark = pytest.mark.requires_rust_extension
IMAGE_DOCUMENT: Final = {"type": "image_url", "image_url": "data:image/png;base64,YWJj"}
REDUCTO_DOCUMENT: Final = {"type": "document_url", "document_url": "reducto://ready.pdf"}
VERTEX: Final = {"vertex_project": "test-project", "vertex_location": "us-central1"}
@dataclass(frozen=True, slots=True)
class Provider:
model: str
document: Mapping[str, object]
arguments: Mapping[str, object]
PROVIDERS: Final = {
"mistral": Provider("mistral/mistral-ocr-latest", OCR_DOCUMENT, {}),
"azure-mistral": Provider("azure_ai/mistral-ocr-latest", OCR_DOCUMENT, {}),
"azure-document-intelligence": Provider("azure_ai/doc-intelligence/prebuilt-layout", OCR_DOCUMENT, {}),
"azure-cohere": Provider("azure_ai/Cohere-parse-v5.0", IMAGE_DOCUMENT, {}),
"cohere": Provider("cohere/parse-v5.0", IMAGE_DOCUMENT, {}),
"vertex-mistral": Provider("vertex_ai/mistral-ocr-latest", OCR_DOCUMENT, VERTEX),
"vertex-deepseek": Provider("vertex_ai/deepseek-ocr", OCR_DOCUMENT, VERTEX),
"reducto-v3": Provider("reducto/parse-v3", REDUCTO_DOCUMENT, {}),
"reducto-legacy": Provider("reducto/parse-legacy", REDUCTO_DOCUMENT, {}),
}
STATUSES: Final = (400, 401, 403, 404, 408, 409, 413, 422, 429, 498, 500, 502, 503, 504, 529)
BODIES: Final = {
"json-object": {"message": "rejected"},
"json-string": "rejected",
"empty-object": {},
"rate-limit-text": {"message": "rate limit reached for requests"},
"context-window-text": {"message": "This model's maximum context length is 10 tokens"},
"content-policy-text": {"message": "content_policy_violation"},
"invalid-request-text": {"error": {"type": "invalid_request_error", "message": "bad field"}},
"model-not-found-text": {"error": {"type": "invalid_request_error", "code": "model_not_found"}},
"timeout-text": {"message": "Request timed out"},
"vertex-quota-text": {"message": "429 Quota exceeded for aiplatform"},
"vertex-api-key-text": {"message": "API key not valid. Please pass a valid API key."},
"cohere-api-token-text": {"message": "invalid api token"},
"cohere-internal-text": {"message": "Internal Server Error"},
}
REPRESENTATIVE_STATUSES: Final = (400, 429, 500)
NO_RETRIES: Final = MappingProxyType({"num_retries": 0})
CALL_OPTIONS: Final = {
"no-retries": NO_RETRIES,
"retries-and-timeout": MappingProxyType({"num_retries": 0, "timeout": 7.5}),
"defaults": MappingProxyType({}),
}
VERTEX_UPSTREAM_CONTENT: Final = tuple(
Divergence(field, "Rust keeps the upstream body and headers that _map_vertex_exception replaces with a stub")
for field in ("response_text", "response_headers")
)
def upstream_divergences(
asynchronous: bool, provider: Provider, options: Mapping[str, object], python: Outcome
) -> tuple[Divergence, ...]:
sync: Final = tuple(
Divergence(field, "Rust sets the request retry count and timeout on sync failures; Python only on async")
for field in ("num_retries", "timeout")
if not asynchronous and options.get(field) is not None
)
vertex: Final = provider.model.startswith("vertex_ai/") and python.response_text == ""
return sync + (VERTEX_UPSTREAM_CONTENT if vertex else ())
def arguments(server: RecordingServer, provider: Provider, **extra: object) -> dict[str, object]:
return {
"model": provider.model,
"document": dict(provider.document),
"api_key": "test-key",
"api_base": server.base_url,
**provider.arguments,
**extra,
}
async def call(
server: RecordingServer, asynchronous: bool, provider: Provider, options: Mapping[str, object]
) -> OCRResponse:
if asynchronous:
return await litellm.aocr(**arguments(server, provider, **options))
return litellm.ocr(**arguments(server, provider, **options))
async def compare_upstream(
server: RecordingServer,
monkeypatch: pytest.MonkeyPatch,
provider: Provider,
response: ResponseSpec,
asynchronous: bool,
options: Mapping[str, object] = NO_RETRIES,
masks: tuple[Masked, ...] = (),
) -> None:
server.expected_requests = None
server.enqueue(response)
python: Final = await run_on(False, monkeypatch, lambda: call(server, asynchronous, provider, options))
server.enqueue(response)
rust: Final = await run_on(True, monkeypatch, lambda: call(server, asynchronous, provider, options))
assert_parity(python, rust, upstream_divergences(asynchronous, provider, options, python), masks)
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("status", STATUSES)
@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys())
async def test_upstream_status_parity(
recording_server: RecordingServer,
monkeypatch: pytest.MonkeyPatch,
provider: Provider,
status: int,
asynchronous: bool,
) -> None:
await compare_upstream(
recording_server,
monkeypatch,
provider,
ResponseSpec(body=BODIES["json-object"], status=status, headers={"retry-after": "7"}),
asynchronous,
)
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("status", REPRESENTATIVE_STATUSES)
@pytest.mark.parametrize("body", BODIES.values(), ids=BODIES.keys())
@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys())
async def test_upstream_body_parity(
recording_server: RecordingServer,
monkeypatch: pytest.MonkeyPatch,
provider: Provider,
body: object,
status: int,
asynchronous: bool,
) -> None:
await compare_upstream(
recording_server, monkeypatch, provider, ResponseSpec(body=body, status=status), asynchronous
)
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("options", CALL_OPTIONS.values(), ids=CALL_OPTIONS.keys())
@pytest.mark.parametrize("status", (401, 500))
@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys())
async def test_call_option_parity(
recording_server: RecordingServer,
monkeypatch: pytest.MonkeyPatch,
provider: Provider,
status: int,
options: Mapping[str, object],
asynchronous: bool,
) -> None:
await compare_upstream(
recording_server,
monkeypatch,
provider,
ResponseSpec(body=BODIES["json-object"], status=status),
asynchronous,
options,
)
def closed_port_base_url() -> str:
with socket.socket() as probe:
probe.bind(("127.0.0.1", 0))
host, port = probe.getsockname()
return f"http://{host}:{port}"
@dataclass(frozen=True, slots=True)
class Transport:
response: ResponseSpec | None
options: Mapping[str, object]
TRANSPORTS: Final = {
"read-timeout": Transport(ResponseSpec(body={}, delay=1.5), {**NO_RETRIES, "timeout": 0.5}),
"connection-refused": Transport(None, NO_RETRIES),
"not-json": Transport(ResponseSpec(body=None, raw=b"<html>bad gateway</html>"), NO_RETRIES),
"missing-fields": Transport(ResponseSpec(body={"unexpected": True}), NO_RETRIES),
"oversized": Transport(ResponseSpec(body={"padding": "x" * 64}), {**NO_RETRIES, "max_response_bytes": 16}),
}
def replaced(pattern: re.Pattern[str], placeholder: str) -> Callable[[object], object]:
return lambda value: pattern.sub(placeholder, value) if isinstance(value, str) else value
ELAPSED: Final = tuple(
Masked(
field,
"both handlers report how long the timed-out call ran",
replaced(re.compile(r"time taken=[0-9.]+ seconds"), "time taken=<elapsed> seconds"),
)
for field in ("text", "message")
)
REFUSED_CONNECTION: Final = re.compile(r"\[Errno \d+\] Connection refused|error sending request")
def refused_connection_divergences(provider: Provider) -> tuple[Divergence, ...]:
reason: Final = "httpx and reqwest word a refused connection differently"
text: Final = tuple(
Divergence(field, reason, replaced(REFUSED_CONNECTION, "<connection refused>")) for field in ("text", "message")
)
vertex: Final = (Divergence("response_text", reason),) if provider.model.startswith("vertex_ai/") else ()
return text + vertex
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("transport", TRANSPORTS.values(), ids=TRANSPORTS.keys())
@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys())
async def test_transport_parity(
recording_server: RecordingServer,
monkeypatch: pytest.MonkeyPatch,
provider: Provider,
transport: Transport,
asynchronous: bool,
) -> None:
if transport.response is not None:
await compare_upstream(
recording_server, monkeypatch, provider, transport.response, asynchronous, transport.options, ELAPSED
)
return
recording_server.expected_requests = 0
unreachable: Final = {**transport.options, "api_base": closed_port_base_url()}
python: Final = await run_on(
False, monkeypatch, lambda: call(recording_server, asynchronous, provider, unreachable)
)
rust: Final = await run_on(True, monkeypatch, lambda: call(recording_server, asynchronous, provider, unreachable))
assert_parity(
python,
rust,
upstream_divergences(asynchronous, provider, unreachable, python) + refused_connection_divergences(provider),
)
@dataclass(frozen=True, slots=True)
class Validation:
document: Callable[[Path], Mapping[str, object]] | None = None
options: Mapping[str, object] = NO_RETRIES
api_key: str | None = "test-key"
def file_document(file: object, **extra: object) -> Mapping[str, object]:
return {"type": "file", "file": file, **extra}
def unreadable_file(directory: Path) -> Mapping[str, object]:
path: Final = directory / "unreadable.pdf"
if not path.exists():
path.write_bytes(b"%PDF-1.4")
path.chmod(0)
return file_document(path)
class RaisingReader:
def read(self) -> bytes:
raise ValueError("reader exploded")
def raising_token_provider() -> str:
raise ValueError("token unavailable")
def url_document(url: str) -> Callable[[Path], Mapping[str, object]]:
return lambda _: {"type": "document_url", "document_url": url}
GENERIC_VALIDATIONS: Final = {
"bad-req-format": Validation(options={**NO_RETRIES, "req_format": "it's-bogus"}),
"empty-file": Validation(lambda _: file_document(b"")),
"missing-file": Validation(lambda directory: file_document(directory / "missing.pdf")),
"unreadable-file": Validation(unreadable_file),
"missing-document-url": Validation(url_document("")),
"invalid-mime-type": Validation(lambda _: file_document(b"abc", mime_type="not a mime type")),
"invalid-data-uri": Validation(url_document("data:application/pdf;base64")),
}
SPECIFIC_VALIDATIONS: Final = {
"cohere-pdf": ("cohere", Validation(url_document(str(OCR_DOCUMENT["document_url"])))),
"azure-cohere-pdf": ("azure-cohere", Validation(url_document(str(OCR_DOCUMENT["document_url"])))),
"reducto-plain-url": ("reducto-v3", Validation(url_document("https://example.com/scan.pdf"))),
"reducto-legacy-plain-url": ("reducto-legacy", Validation(url_document("https://example.com/scan.pdf"))),
"invalid-pages": ("azure-document-intelligence", Validation(options={**NO_RETRIES, "pages": [-1]})),
"missing-azure-ai-credentials": ("azure-mistral", Validation(api_key=None)),
"missing-document-intelligence-credentials": ("azure-document-intelligence", Validation(api_key=None)),
"missing-reducto-key": ("reducto-v3", Validation(api_key=None)),
"raising-file-reader": ("mistral", Validation(lambda _: file_document(RaisingReader()))),
"raising-token-provider": (
"azure-mistral",
Validation(options={**NO_RETRIES, "azure_ad_token_provider": raising_token_provider}, api_key=None),
),
}
VALIDATION_ROWS: Final = {
**{
f"{name}-{provider}": (PROVIDERS[provider], validation)
for name, validation in GENERIC_VALIDATIONS.items()
for provider in PROVIDERS
},
**{name: (PROVIDERS[provider], validation) for name, (provider, validation) in SPECIFIC_VALIDATIONS.items()},
"invalid-provider": (Provider("not_a_provider/ocr-model", OCR_DOCUMENT, {}), Validation()),
}
CREDENTIAL_VARIABLES: Final = ("AZURE_DOCUMENT_INTELLIGENCE_API_KEY", "REDUCTO_API_KEY", "MISTRAL_API_KEY")
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("row", VALIDATION_ROWS.values(), ids=VALIDATION_ROWS.keys())
async def test_validation_parity(
recording_server: RecordingServer,
monkeypatch: pytest.MonkeyPatch,
isolated_azure_auth: None,
tmp_path: Path,
row: tuple[Provider, Validation],
asynchronous: bool,
) -> None:
provider, validation = row
for variable in CREDENTIAL_VARIABLES:
monkeypatch.delenv(variable, raising=False)
recording_server.expected_requests = None
def invoke() -> Awaitable[OCRResponse]:
document: Final = provider.document if validation.document is None else validation.document(tmp_path)
options: Final = {**validation.options, "document": dict(document), "api_key": validation.api_key}
return call(recording_server, asynchronous, provider, options)
python: Final = await run_on(False, monkeypatch, invoke)
python_requests: Final = len(recording_server.requests)
rust: Final = await run_on(True, monkeypatch, invoke)
assert len(recording_server.requests) - python_requests == python_requests
assert_parity(python, rust, upstream_divergences(asynchronous, provider, validation.options, python))

View file

@ -24,13 +24,6 @@ from tests.test_litellm_rust.support.requests import (
pytestmark = pytest.mark.requires_rust_extension
@pytest.fixture(params=[False, True], ids=["python", "rust"])
def ocr_backend(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> bool:
enabled: Final = bool(request.param)
monkeypatch.setenv("LITELLM_RUST", "1" if enabled else "0")
return enabled
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_ocr_contract_upstream_status(

View file

@ -0,0 +1,98 @@
from collections.abc import Mapping
from dataclasses import dataclass
from types import MappingProxyType
from typing import Final
import pytest
from fastapi.testclient import TestClient
from litellm import Router
from litellm.proxy import proxy_server
from tests.test_litellm_rust.support.parity import assert_parity
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
from tests.test_litellm_rust.support.requests import OCR_DOCUMENT, OCR_MODEL, OCR_RESPONSE
pytestmark = pytest.mark.requires_rust_extension
MODEL_GROUP: Final = "ocr"
ROUTER_RETRIES: Final = 1
@dataclass(frozen=True, slots=True)
class Row:
upstream: ResponseSpec
document: Mapping[str, object] = MappingProxyType(dict(OCR_DOCUMENT))
deployment: Mapping[str, object] = MappingProxyType({})
@dataclass(frozen=True, slots=True)
class ProxyOutcome:
status: int
error_type: object
error_code: object
upstream_requests: int
cooled_down: int
timeout_header: bool
markdown: object
ROWS: Final = {
"success": Row(ResponseSpec(body=OCR_RESPONSE)),
"unauthorized": Row(ResponseSpec(body={"message": "rejected"}, status=401)),
"not-found": Row(ResponseSpec(body={"message": "no such model"}, status=404)),
"rate-limited": Row(ResponseSpec(body={"message": "slow down"}, status=429, headers={"retry-after": "0"})),
"provider-failure": Row(ResponseSpec(body={"message": "provider unavailable"}, status=500)),
"read-timeout": Row(ResponseSpec(body=OCR_RESPONSE, delay=1.5), deployment=MappingProxyType({"timeout": 0.5})),
"not-json": Row(ResponseSpec(body=None, raw=b"<html>bad gateway</html>")),
"missing-document-url": Row(
ResponseSpec(body=OCR_RESPONSE), document=MappingProxyType({"type": "document_url", "document_url": ""})
),
}
def observe(server: RecordingServer, row: Row, monkeypatch: pytest.MonkeyPatch, backend: bool) -> ProxyOutcome:
monkeypatch.setenv("LITELLM_RUST", "1" if backend else "0")
server.default_response = row.upstream
before: Final = len(server.requests)
router: Final = Router(
model_list=[
{
"model_name": MODEL_GROUP,
"litellm_params": {
"model": OCR_MODEL,
"api_key": "test-key",
"api_base": server.base_url,
**row.deployment,
},
}
],
num_retries=ROUTER_RETRIES,
retry_after=0,
)
monkeypatch.setattr(proxy_server, "llm_router", router)
monkeypatch.setattr(proxy_server, "master_key", None)
monkeypatch.setattr(proxy_server, "general_settings", {})
with TestClient(proxy_server.app, raise_server_exceptions=False) as client:
response: Final = client.post("/v1/ocr", json={"model": MODEL_GROUP, "document": dict(row.document)})
body: Final = response.json()
error: Final = body.get("error") if isinstance(body, dict) else None
pages: Final = body.get("pages") if isinstance(body, dict) else None
if response.status_code == 200:
assert response.headers.get("x-litellm-rust") == ("true" if backend else None)
return ProxyOutcome(
status=response.status_code,
error_type=None if not isinstance(error, dict) else error.get("type"),
error_code=None if not isinstance(error, dict) else error.get("code"),
upstream_requests=len(server.requests) - before,
cooled_down=len(router.cooldown_cache.get_active_cooldowns(router.get_model_ids(), parent_otel_span=None)),
timeout_header="x-litellm-timeout" in response.headers,
markdown=pages[0].get("markdown") if isinstance(pages, list) and pages else None,
)
@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys())
def test_router_and_proxy_parity(recording_server: RecordingServer, monkeypatch: pytest.MonkeyPatch, row: Row) -> None:
recording_server.expected_requests = None
python: Final = observe(recording_server, row, monkeypatch, backend=False)
rust: Final = observe(recording_server, row, monkeypatch, backend=True)
assert_parity(python, rust)

View file

@ -0,0 +1,142 @@
from collections.abc import Awaitable, Callable, Mapping
from dataclasses import dataclass, fields, replace
from types import MappingProxyType
from typing import TYPE_CHECKING, Final, TypeVar
import httpx
import pytest
from litellm.llms.base_llm.ocr.transformation import OCRResponse
if TYPE_CHECKING:
from _typeshed import DataclassInstance
VOLATILE_HEADERS: Final = frozenset({"date", "server"})
@dataclass(frozen=True, slots=True)
class Outcome:
kind: str
error_class: str | None = None
text: str | None = None
message: str | None = None
status_code: object = None
llm_provider: object = None
model: object = None
litellm_debug_info: object = None
num_retries: object = None
max_retries: object = None
timeout: object = None
response_status: int | None = None
response_text: str | None = None
response_headers: Mapping[str, str] | None = None
litellm_response_headers: Mapping[str, str] | None = None
result: object = None
def _headers(value: object) -> Mapping[str, str] | None:
if not isinstance(value, httpx.Headers | Mapping):
return None
return MappingProxyType(
{str(name).lower(): str(item) for name, item in value.items() if str(name).lower() not in VOLATILE_HEADERS}
)
def _response(error: BaseException) -> httpx.Response | None:
try:
response: Final = getattr(error, "response", None)
except RuntimeError:
return None
return response if isinstance(response, httpx.Response) else None
def failure_outcome(error: BaseException) -> Outcome:
response: Final = _response(error)
return Outcome(
kind="error",
error_class=f"{type(error).__module__}.{type(error).__qualname__}",
text=str(error),
message=getattr(error, "message", None),
status_code=getattr(error, "status_code", None),
llm_provider=getattr(error, "llm_provider", None),
model=getattr(error, "model", None),
litellm_debug_info=getattr(error, "litellm_debug_info", None),
num_retries=getattr(error, "num_retries", None),
max_retries=getattr(error, "max_retries", None),
timeout=getattr(error, "timeout", None),
response_status=None if response is None else response.status_code,
response_text=None if response is None else response.text,
response_headers=None if response is None else _headers(response.headers),
litellm_response_headers=_headers(getattr(error, "litellm_response_headers", None)),
)
def success_outcome(response: OCRResponse) -> Outcome:
return Outcome(kind="ok", result=response.model_dump())
@dataclass(frozen=True, slots=True)
class Divergence:
"""A field that differs by design. With `normalize`, both sides are compared after it
instead of the field being dropped."""
field: str
reason: str
normalize: Callable[[object], object] | None = None
@dataclass(frozen=True, slots=True)
class Masked:
"""A field that holds a run-dependent value on both sides, compared after `mask`."""
field: str
reason: str
mask: Callable[[object], object]
async def observe(call: Callable[[], Awaitable[OCRResponse]]) -> Outcome:
try:
return success_outcome(await call())
except Exception as error:
return failure_outcome(error)
async def run_on(backend: bool, monkeypatch: pytest.MonkeyPatch, call: Callable[[], Awaitable[OCRResponse]]) -> Outcome:
monkeypatch.setenv("LITELLM_RUST", "1" if backend else "0")
return await observe(call)
ObservedT = TypeVar("ObservedT", bound="DataclassInstance")
def without(outcome: ObservedT, divergences: tuple[Divergence, ...], masks: tuple[Masked, ...] = ()) -> ObservedT:
compared: Final = {
**{
divergence.field: None
if divergence.normalize is None
else divergence.normalize(getattr(outcome, divergence.field))
for divergence in divergences
},
**{masked.field: masked.mask(getattr(outcome, masked.field)) for masked in masks},
}
return replace(outcome, **compared)
def assert_parity(
python: ObservedT, rust: ObservedT, divergences: tuple[Divergence, ...] = (), masks: tuple[Masked, ...] = ()
) -> None:
known: Final = frozenset(divergence.field for divergence in divergences) | {masked.field for masked in masks}
unknown: Final = known - {field.name for field in fields(python)}
assert not unknown, f"divergence registry names unknown fields: {sorted(unknown)}"
for divergence in divergences:
assert getattr(python, divergence.field) != getattr(rust, divergence.field), (
f"registered divergence on {divergence.field!r} no longer diverges; remove it: {divergence.reason}"
)
expected: Final = without(python, divergences, masks)
actual: Final = without(rust, divergences, masks)
differences: Final = {
field.name: {"python": getattr(expected, field.name), "rust": getattr(actual, field.name)}
for field in fields(python)
if getattr(expected, field.name) != getattr(actual, field.name)
}
assert not differences, differences

View file

@ -26,8 +26,11 @@ class ResponseSpec:
headers: dict[str, str] = field(default_factory=dict)
delay: float = 0
events: tuple[tuple[str, object], ...] = ()
raw: bytes | None = None
def payloads(self) -> tuple[bytes, ...]:
if self.raw is not None:
return (self.raw,)
if not self.events:
return (json.dumps(self.body).encode(),)
return tuple(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() for event, data in self.events)