mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
test(rust): pin Vertex OCR hosts for global and multi-region locations
The Python path picks the host with get_vertex_base_url: global uses
aiplatform.googleapis.com and us/eu use the .rep. hosts. The Rust config
always builds {location}-aiplatform.googleapis.com, so these cases fail
until it matches
This commit is contained in:
parent
ff7dc86947
commit
c20ae5fffb
10 changed files with 882 additions and 11 deletions
|
|
@ -204,15 +204,27 @@ fn validate_location(location: &str) -> Result<(), Error> {
|
|||
#[cfg(test)]
|
||||
mod tests {
|
||||
|
||||
use rstest::rstest;
|
||||
|
||||
use super::VertexAiOcrConfig;
|
||||
|
||||
#[test]
|
||||
fn endpoint_uses_location_project_and_model() {
|
||||
/// The hosts `get_vertex_base_url` in `litellm/llms/vertex_ai/common_utils.py` picks.
|
||||
#[rstest]
|
||||
#[case::regional("europe-west4", "https://europe-west4-aiplatform.googleapis.com")]
|
||||
#[case::global("global", "https://aiplatform.googleapis.com")]
|
||||
#[case::multi_region_us("us", "https://aiplatform.us.rep.googleapis.com")]
|
||||
#[case::multi_region_eu("eu", "https://aiplatform.eu.rep.googleapis.com")]
|
||||
fn endpoint_uses_the_location_host_project_and_model(
|
||||
#[case] location: &str,
|
||||
#[case] host: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
VertexAiOcrConfig
|
||||
.build_ocr_url(None, "proj-1", "europe-west4", "mistral-ocr-maas")
|
||||
.build_ocr_url(None, "proj-1", location, "mistral-ocr-maas")
|
||||
.unwrap(),
|
||||
"https://europe-west4-aiplatform.googleapis.com/v1/projects/proj-1/locations/europe-west4/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
|
||||
format!(
|
||||
"{host}/v1/projects/proj-1/locations/{location}/publishers/mistralai/models/mistral-ocr-maas:rawPredict"
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -41,6 +41,13 @@ async def isolate_ocr_test_state() -> AsyncIterator[None]:
|
|||
await GLOBAL_LOGGING_WORKER.stop()
|
||||
|
||||
|
||||
@pytest.fixture(params=[False, True], ids=["python", "rust"])
|
||||
def ocr_backend(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> bool:
|
||||
enabled: Final = bool(request.param)
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if enabled else "0")
|
||||
return enabled
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recording_server() -> Generator[RecordingServer]:
|
||||
with recording_service() as server:
|
||||
|
|
|
|||
12
tests/test_litellm_rust/ocr/conftest.py
Normal file
12
tests/test_litellm_rust/ocr/conftest.py
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.test_litellm_rust.ocr.parity_todo import TODO
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def parity_todo(request: pytest.FixtureRequest) -> None:
|
||||
reason: Final = TODO.get(request.node.name)
|
||||
if reason is not None:
|
||||
request.applymarker(pytest.mark.xfail(strict=True, reason=reason))
|
||||
229
tests/test_litellm_rust/ocr/parity_todo.py
Normal file
229
tests/test_litellm_rust/ocr/parity_todo.py
Normal file
|
|
@ -0,0 +1,229 @@
|
|||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
TODO: Final = MappingProxyType(
|
||||
{
|
||||
"test_transport_parity[azure-cohere-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[azure-cohere-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[azure-cohere-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[azure-cohere-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[azure-cohere-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[azure-cohere-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[azure-cohere-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[azure-cohere-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[azure-cohere-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[azure-document-intelligence-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[azure-document-intelligence-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[azure-document-intelligence-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[azure-document-intelligence-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[azure-document-intelligence-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[azure-document-intelligence-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[azure-document-intelligence-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[azure-document-intelligence-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[azure-document-intelligence-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[azure-document-intelligence-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[azure-mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[azure-mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[azure-mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[azure-mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[azure-mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[azure-mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[azure-mistral-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[azure-mistral-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[azure-mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[azure-mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[cohere-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[cohere-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[cohere-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[cohere-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[cohere-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[cohere-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[cohere-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[cohere-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[cohere-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[mistral-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[mistral-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[reducto-legacy-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[reducto-legacy-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[reducto-legacy-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[reducto-legacy-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[reducto-legacy-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[reducto-legacy-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[reducto-legacy-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[reducto-legacy-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[reducto-legacy-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[reducto-v3-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[reducto-v3-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[reducto-v3-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[reducto-v3-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[reducto-v3-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[reducto-v3-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[reducto-v3-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[reducto-v3-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[reducto-v3-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[vertex-deepseek-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[vertex-deepseek-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[vertex-deepseek-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[vertex-deepseek-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[vertex-deepseek-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[vertex-deepseek-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[vertex-deepseek-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[vertex-deepseek-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[vertex-deepseek-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[vertex-deepseek-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[vertex-mistral-connection-refused-async]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[vertex-mistral-connection-refused-sync]": "4.2: Python reports a refused connection as a 500 from the provider",
|
||||
"test_transport_parity[vertex-mistral-missing-fields-async]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[vertex-mistral-missing-fields-sync]": "4.2: responses missing fields do not map like the Python transformation",
|
||||
"test_transport_parity[vertex-mistral-not-json-async]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[vertex-mistral-not-json-sync]": "4.2: unparseable responses carry the Rust parser's message",
|
||||
"test_transport_parity[vertex-mistral-oversized-async]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[vertex-mistral-oversized-sync]": "4.2: Python has no response size limit",
|
||||
"test_transport_parity[vertex-mistral-read-timeout-async]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_transport_parity[vertex-mistral-read-timeout-sync]": "4.2: timeouts carry a Rust message instead of the Python handler's",
|
||||
"test_validation_parity[azure-cohere-pdf-async]": "4.4: Cohere image-only text differs",
|
||||
"test_validation_parity[azure-cohere-pdf-sync]": "4.4: Cohere image-only text differs",
|
||||
"test_validation_parity[bad-req-format-azure-cohere-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-azure-cohere-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-azure-document-intelligence-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-azure-document-intelligence-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-azure-mistral-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-azure-mistral-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-cohere-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-cohere-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-mistral-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-mistral-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-reducto-legacy-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-reducto-legacy-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-reducto-v3-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-reducto-v3-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-vertex-deepseek-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-vertex-deepseek-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-vertex-mistral-async]": "4.4: req_format text differs",
|
||||
"test_validation_parity[bad-req-format-vertex-mistral-sync]": "4.4: req_format text differs",
|
||||
"test_validation_parity[cohere-pdf-async]": "4.4: Cohere image-only text differs",
|
||||
"test_validation_parity[cohere-pdf-sync]": "4.4: Cohere image-only text differs",
|
||||
"test_validation_parity[empty-file-azure-cohere-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-azure-cohere-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-azure-document-intelligence-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-azure-document-intelligence-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-azure-mistral-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-azure-mistral-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-cohere-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-cohere-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-mistral-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-mistral-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-reducto-legacy-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-reducto-legacy-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-reducto-v3-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-reducto-v3-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-vertex-deepseek-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-vertex-deepseek-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-vertex-mistral-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[empty-file-vertex-mistral-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-data-uri-azure-cohere-async]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-azure-cohere-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-azure-document-intelligence-async]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-azure-document-intelligence-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-azure-mistral-async]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-azure-mistral-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-cohere-async]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-cohere-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-mistral-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-reducto-legacy-async]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-reducto-legacy-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-reducto-v3-async]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-reducto-v3-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-vertex-deepseek-async]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-vertex-deepseek-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-vertex-mistral-async]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-data-uri-vertex-mistral-sync]": "4.4: Rust rejects data URIs Python forwards",
|
||||
"test_validation_parity[invalid-mime-type-azure-cohere-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-azure-cohere-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-azure-document-intelligence-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-azure-document-intelligence-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-azure-mistral-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-azure-mistral-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-cohere-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-cohere-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-mistral-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-mistral-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-reducto-legacy-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-reducto-legacy-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-reducto-v3-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-reducto-v3-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-vertex-deepseek-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-vertex-deepseek-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-vertex-mistral-async]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-mime-type-vertex-mistral-sync]": "4.4: local ValueError classification",
|
||||
"test_validation_parity[invalid-pages-async]": "4.4: pages validation text differs",
|
||||
"test_validation_parity[invalid-pages-sync]": "4.4: pages validation text differs",
|
||||
"test_validation_parity[invalid-provider-async]": "4.4: invalid provider text differs",
|
||||
"test_validation_parity[invalid-provider-sync]": "4.4: invalid provider text differs",
|
||||
"test_validation_parity[missing-azure-ai-credentials-async]": "4.4: credential text differs",
|
||||
"test_validation_parity[missing-azure-ai-credentials-sync]": "4.4: credential text differs",
|
||||
"test_validation_parity[missing-document-intelligence-credentials-async]": "4.4: credential text differs",
|
||||
"test_validation_parity[missing-document-intelligence-credentials-sync]": "4.4: credential text differs",
|
||||
"test_validation_parity[missing-document-url-azure-cohere-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-azure-cohere-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-azure-document-intelligence-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-azure-document-intelligence-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-azure-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-azure-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-cohere-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-cohere-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-reducto-legacy-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-reducto-legacy-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-reducto-v3-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-reducto-v3-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-vertex-deepseek-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-vertex-deepseek-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-vertex-mistral-async]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-document-url-vertex-mistral-sync]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
"test_validation_parity[missing-file-cohere-async]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-cohere-sync]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-reducto-legacy-async]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-reducto-legacy-sync]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-reducto-v3-async]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-reducto-v3-sync]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-vertex-deepseek-async]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-vertex-deepseek-sync]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-vertex-mistral-async]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-file-vertex-mistral-sync]": "4.4: local FileNotFoundError classification",
|
||||
"test_validation_parity[missing-reducto-key-async]": "4.4: credential error classification",
|
||||
"test_validation_parity[missing-reducto-key-sync]": "4.4: credential error classification",
|
||||
"test_validation_parity[reducto-legacy-plain-url-async]": "4.4: Reducto source text differs",
|
||||
"test_validation_parity[reducto-legacy-plain-url-sync]": "4.4: Reducto source text differs",
|
||||
"test_validation_parity[reducto-plain-url-async]": "4.4: Reducto source text differs",
|
||||
"test_validation_parity[reducto-plain-url-sync]": "4.4: Reducto source text differs",
|
||||
"test_validation_parity[unreadable-file-azure-cohere-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-azure-cohere-sync]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-azure-document-intelligence-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-azure-document-intelligence-sync]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-azure-mistral-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-azure-mistral-sync]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-cohere-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-cohere-sync]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-mistral-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-mistral-sync]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-reducto-legacy-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-reducto-legacy-sync]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-reducto-v3-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-reducto-v3-sync]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-vertex-deepseek-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-vertex-deepseek-sync]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-vertex-mistral-async]": "4.4: local OSError classification",
|
||||
"test_validation_parity[unreadable-file-vertex-mistral-sync]": "4.4: local OSError classification",
|
||||
"test_router_and_proxy_parity[missing-document-url]": "4.4: Python rejects an empty document URL before provider resolution",
|
||||
}
|
||||
)
|
||||
|
|
@ -44,6 +44,24 @@ async def call_native_aocr_with_callbacks(server: RecordingServer, callbacks: li
|
|||
return await call_native_aocr(server, callbacks=callbacks, **kwargs)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_ocr_contract_post_call_event_carries_the_api_key_and_only_the_request_body(
|
||||
ocr_server: RecordingServer, ocr_backend: bool, asynchronous: bool
|
||||
) -> None:
|
||||
observations: Final = []
|
||||
|
||||
class Observe(CustomLogger):
|
||||
def log_post_api_call(self, kwargs, response_obj, start_time, end_time):
|
||||
observations.append((kwargs["api_key"], copy.deepcopy(kwargs["additional_args"])))
|
||||
|
||||
await call_native(ocr_server, asynchronous, callbacks=[Observe()])
|
||||
|
||||
assert observations == [
|
||||
("test-key", {"complete_input_dict": {"model": "mistral-ocr-latest", "document": OCR_DOCUMENT}})
|
||||
]
|
||||
|
||||
|
||||
def test_native_ocr_pre_call_callback_receives_transformed_provider_request(ocr_server: RecordingServer) -> None:
|
||||
observations: Final = []
|
||||
|
||||
|
|
|
|||
357
tests/test_litellm_rust/ocr/test_parity.py
Normal file
357
tests/test_litellm_rust/ocr/test_parity.py
Normal file
|
|
@ -0,0 +1,357 @@
|
|||
import re
|
||||
import socket
|
||||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
|
||||
import litellm
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRResponse
|
||||
from tests.test_litellm_rust.support.parity import Divergence, Masked, Outcome, assert_parity, run_on
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
|
||||
from tests.test_litellm_rust.support.requests import OCR_DOCUMENT
|
||||
|
||||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
||||
IMAGE_DOCUMENT: Final = {"type": "image_url", "image_url": "data:image/png;base64,YWJj"}
|
||||
REDUCTO_DOCUMENT: Final = {"type": "document_url", "document_url": "reducto://ready.pdf"}
|
||||
VERTEX: Final = {"vertex_project": "test-project", "vertex_location": "us-central1"}
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Provider:
|
||||
model: str
|
||||
document: Mapping[str, object]
|
||||
arguments: Mapping[str, object]
|
||||
|
||||
|
||||
PROVIDERS: Final = {
|
||||
"mistral": Provider("mistral/mistral-ocr-latest", OCR_DOCUMENT, {}),
|
||||
"azure-mistral": Provider("azure_ai/mistral-ocr-latest", OCR_DOCUMENT, {}),
|
||||
"azure-document-intelligence": Provider("azure_ai/doc-intelligence/prebuilt-layout", OCR_DOCUMENT, {}),
|
||||
"azure-cohere": Provider("azure_ai/Cohere-parse-v5.0", IMAGE_DOCUMENT, {}),
|
||||
"cohere": Provider("cohere/parse-v5.0", IMAGE_DOCUMENT, {}),
|
||||
"vertex-mistral": Provider("vertex_ai/mistral-ocr-latest", OCR_DOCUMENT, VERTEX),
|
||||
"vertex-deepseek": Provider("vertex_ai/deepseek-ocr", OCR_DOCUMENT, VERTEX),
|
||||
"reducto-v3": Provider("reducto/parse-v3", REDUCTO_DOCUMENT, {}),
|
||||
"reducto-legacy": Provider("reducto/parse-legacy", REDUCTO_DOCUMENT, {}),
|
||||
}
|
||||
|
||||
STATUSES: Final = (400, 401, 403, 404, 408, 409, 413, 422, 429, 498, 500, 502, 503, 504, 529)
|
||||
|
||||
BODIES: Final = {
|
||||
"json-object": {"message": "rejected"},
|
||||
"json-string": "rejected",
|
||||
"empty-object": {},
|
||||
"rate-limit-text": {"message": "rate limit reached for requests"},
|
||||
"context-window-text": {"message": "This model's maximum context length is 10 tokens"},
|
||||
"content-policy-text": {"message": "content_policy_violation"},
|
||||
"invalid-request-text": {"error": {"type": "invalid_request_error", "message": "bad field"}},
|
||||
"model-not-found-text": {"error": {"type": "invalid_request_error", "code": "model_not_found"}},
|
||||
"timeout-text": {"message": "Request timed out"},
|
||||
"vertex-quota-text": {"message": "429 Quota exceeded for aiplatform"},
|
||||
"vertex-api-key-text": {"message": "API key not valid. Please pass a valid API key."},
|
||||
"cohere-api-token-text": {"message": "invalid api token"},
|
||||
"cohere-internal-text": {"message": "Internal Server Error"},
|
||||
}
|
||||
|
||||
REPRESENTATIVE_STATUSES: Final = (400, 429, 500)
|
||||
|
||||
NO_RETRIES: Final = MappingProxyType({"num_retries": 0})
|
||||
CALL_OPTIONS: Final = {
|
||||
"no-retries": NO_RETRIES,
|
||||
"retries-and-timeout": MappingProxyType({"num_retries": 0, "timeout": 7.5}),
|
||||
"defaults": MappingProxyType({}),
|
||||
}
|
||||
|
||||
VERTEX_UPSTREAM_CONTENT: Final = tuple(
|
||||
Divergence(field, "Rust keeps the upstream body and headers that _map_vertex_exception replaces with a stub")
|
||||
for field in ("response_text", "response_headers")
|
||||
)
|
||||
|
||||
|
||||
def upstream_divergences(
|
||||
asynchronous: bool, provider: Provider, options: Mapping[str, object], python: Outcome
|
||||
) -> tuple[Divergence, ...]:
|
||||
sync: Final = tuple(
|
||||
Divergence(field, "Rust sets the request retry count and timeout on sync failures; Python only on async")
|
||||
for field in ("num_retries", "timeout")
|
||||
if not asynchronous and options.get(field) is not None
|
||||
)
|
||||
vertex: Final = provider.model.startswith("vertex_ai/") and python.response_text == ""
|
||||
return sync + (VERTEX_UPSTREAM_CONTENT if vertex else ())
|
||||
|
||||
|
||||
def arguments(server: RecordingServer, provider: Provider, **extra: object) -> dict[str, object]:
|
||||
return {
|
||||
"model": provider.model,
|
||||
"document": dict(provider.document),
|
||||
"api_key": "test-key",
|
||||
"api_base": server.base_url,
|
||||
**provider.arguments,
|
||||
**extra,
|
||||
}
|
||||
|
||||
|
||||
async def call(
|
||||
server: RecordingServer, asynchronous: bool, provider: Provider, options: Mapping[str, object]
|
||||
) -> OCRResponse:
|
||||
if asynchronous:
|
||||
return await litellm.aocr(**arguments(server, provider, **options))
|
||||
return litellm.ocr(**arguments(server, provider, **options))
|
||||
|
||||
|
||||
async def compare_upstream(
|
||||
server: RecordingServer,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
provider: Provider,
|
||||
response: ResponseSpec,
|
||||
asynchronous: bool,
|
||||
options: Mapping[str, object] = NO_RETRIES,
|
||||
masks: tuple[Masked, ...] = (),
|
||||
) -> None:
|
||||
server.expected_requests = None
|
||||
server.enqueue(response)
|
||||
python: Final = await run_on(False, monkeypatch, lambda: call(server, asynchronous, provider, options))
|
||||
server.enqueue(response)
|
||||
rust: Final = await run_on(True, monkeypatch, lambda: call(server, asynchronous, provider, options))
|
||||
assert_parity(python, rust, upstream_divergences(asynchronous, provider, options, python), masks)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("status", STATUSES)
|
||||
@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys())
|
||||
async def test_upstream_status_parity(
|
||||
recording_server: RecordingServer,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
provider: Provider,
|
||||
status: int,
|
||||
asynchronous: bool,
|
||||
) -> None:
|
||||
await compare_upstream(
|
||||
recording_server,
|
||||
monkeypatch,
|
||||
provider,
|
||||
ResponseSpec(body=BODIES["json-object"], status=status, headers={"retry-after": "7"}),
|
||||
asynchronous,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("status", REPRESENTATIVE_STATUSES)
|
||||
@pytest.mark.parametrize("body", BODIES.values(), ids=BODIES.keys())
|
||||
@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys())
|
||||
async def test_upstream_body_parity(
|
||||
recording_server: RecordingServer,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
provider: Provider,
|
||||
body: object,
|
||||
status: int,
|
||||
asynchronous: bool,
|
||||
) -> None:
|
||||
await compare_upstream(
|
||||
recording_server, monkeypatch, provider, ResponseSpec(body=body, status=status), asynchronous
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("options", CALL_OPTIONS.values(), ids=CALL_OPTIONS.keys())
|
||||
@pytest.mark.parametrize("status", (401, 500))
|
||||
@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys())
|
||||
async def test_call_option_parity(
|
||||
recording_server: RecordingServer,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
provider: Provider,
|
||||
status: int,
|
||||
options: Mapping[str, object],
|
||||
asynchronous: bool,
|
||||
) -> None:
|
||||
await compare_upstream(
|
||||
recording_server,
|
||||
monkeypatch,
|
||||
provider,
|
||||
ResponseSpec(body=BODIES["json-object"], status=status),
|
||||
asynchronous,
|
||||
options,
|
||||
)
|
||||
|
||||
|
||||
def closed_port_base_url() -> str:
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
host, port = probe.getsockname()
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Transport:
|
||||
response: ResponseSpec | None
|
||||
options: Mapping[str, object]
|
||||
|
||||
|
||||
TRANSPORTS: Final = {
|
||||
"read-timeout": Transport(ResponseSpec(body={}, delay=1.5), {**NO_RETRIES, "timeout": 0.5}),
|
||||
"connection-refused": Transport(None, NO_RETRIES),
|
||||
"not-json": Transport(ResponseSpec(body=None, raw=b"<html>bad gateway</html>"), NO_RETRIES),
|
||||
"missing-fields": Transport(ResponseSpec(body={"unexpected": True}), NO_RETRIES),
|
||||
"oversized": Transport(ResponseSpec(body={"padding": "x" * 64}), {**NO_RETRIES, "max_response_bytes": 16}),
|
||||
}
|
||||
|
||||
|
||||
def replaced(pattern: re.Pattern[str], placeholder: str) -> Callable[[object], object]:
|
||||
return lambda value: pattern.sub(placeholder, value) if isinstance(value, str) else value
|
||||
|
||||
|
||||
ELAPSED: Final = tuple(
|
||||
Masked(
|
||||
field,
|
||||
"both handlers report how long the timed-out call ran",
|
||||
replaced(re.compile(r"time taken=[0-9.]+ seconds"), "time taken=<elapsed> seconds"),
|
||||
)
|
||||
for field in ("text", "message")
|
||||
)
|
||||
|
||||
REFUSED_CONNECTION: Final = re.compile(r"\[Errno \d+\] Connection refused|error sending request")
|
||||
|
||||
|
||||
def refused_connection_divergences(provider: Provider) -> tuple[Divergence, ...]:
|
||||
reason: Final = "httpx and reqwest word a refused connection differently"
|
||||
text: Final = tuple(
|
||||
Divergence(field, reason, replaced(REFUSED_CONNECTION, "<connection refused>")) for field in ("text", "message")
|
||||
)
|
||||
vertex: Final = (Divergence("response_text", reason),) if provider.model.startswith("vertex_ai/") else ()
|
||||
return text + vertex
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("transport", TRANSPORTS.values(), ids=TRANSPORTS.keys())
|
||||
@pytest.mark.parametrize("provider", PROVIDERS.values(), ids=PROVIDERS.keys())
|
||||
async def test_transport_parity(
|
||||
recording_server: RecordingServer,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
provider: Provider,
|
||||
transport: Transport,
|
||||
asynchronous: bool,
|
||||
) -> None:
|
||||
if transport.response is not None:
|
||||
await compare_upstream(
|
||||
recording_server, monkeypatch, provider, transport.response, asynchronous, transport.options, ELAPSED
|
||||
)
|
||||
return
|
||||
recording_server.expected_requests = 0
|
||||
unreachable: Final = {**transport.options, "api_base": closed_port_base_url()}
|
||||
python: Final = await run_on(
|
||||
False, monkeypatch, lambda: call(recording_server, asynchronous, provider, unreachable)
|
||||
)
|
||||
rust: Final = await run_on(True, monkeypatch, lambda: call(recording_server, asynchronous, provider, unreachable))
|
||||
assert_parity(
|
||||
python,
|
||||
rust,
|
||||
upstream_divergences(asynchronous, provider, unreachable, python) + refused_connection_divergences(provider),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Validation:
|
||||
document: Callable[[Path], Mapping[str, object]] | None = None
|
||||
options: Mapping[str, object] = NO_RETRIES
|
||||
api_key: str | None = "test-key"
|
||||
|
||||
|
||||
def file_document(file: object, **extra: object) -> Mapping[str, object]:
|
||||
return {"type": "file", "file": file, **extra}
|
||||
|
||||
|
||||
def unreadable_file(directory: Path) -> Mapping[str, object]:
|
||||
path: Final = directory / "unreadable.pdf"
|
||||
if not path.exists():
|
||||
path.write_bytes(b"%PDF-1.4")
|
||||
path.chmod(0)
|
||||
return file_document(path)
|
||||
|
||||
|
||||
class RaisingReader:
|
||||
def read(self) -> bytes:
|
||||
raise ValueError("reader exploded")
|
||||
|
||||
|
||||
def raising_token_provider() -> str:
|
||||
raise ValueError("token unavailable")
|
||||
|
||||
|
||||
def url_document(url: str) -> Callable[[Path], Mapping[str, object]]:
|
||||
return lambda _: {"type": "document_url", "document_url": url}
|
||||
|
||||
|
||||
GENERIC_VALIDATIONS: Final = {
|
||||
"bad-req-format": Validation(options={**NO_RETRIES, "req_format": "it's-bogus"}),
|
||||
"empty-file": Validation(lambda _: file_document(b"")),
|
||||
"missing-file": Validation(lambda directory: file_document(directory / "missing.pdf")),
|
||||
"unreadable-file": Validation(unreadable_file),
|
||||
"missing-document-url": Validation(url_document("")),
|
||||
"invalid-mime-type": Validation(lambda _: file_document(b"abc", mime_type="not a mime type")),
|
||||
"invalid-data-uri": Validation(url_document("data:application/pdf;base64")),
|
||||
}
|
||||
|
||||
SPECIFIC_VALIDATIONS: Final = {
|
||||
"cohere-pdf": ("cohere", Validation(url_document(str(OCR_DOCUMENT["document_url"])))),
|
||||
"azure-cohere-pdf": ("azure-cohere", Validation(url_document(str(OCR_DOCUMENT["document_url"])))),
|
||||
"reducto-plain-url": ("reducto-v3", Validation(url_document("https://example.com/scan.pdf"))),
|
||||
"reducto-legacy-plain-url": ("reducto-legacy", Validation(url_document("https://example.com/scan.pdf"))),
|
||||
"invalid-pages": ("azure-document-intelligence", Validation(options={**NO_RETRIES, "pages": [-1]})),
|
||||
"missing-azure-ai-credentials": ("azure-mistral", Validation(api_key=None)),
|
||||
"missing-document-intelligence-credentials": ("azure-document-intelligence", Validation(api_key=None)),
|
||||
"missing-reducto-key": ("reducto-v3", Validation(api_key=None)),
|
||||
"raising-file-reader": ("mistral", Validation(lambda _: file_document(RaisingReader()))),
|
||||
"raising-token-provider": (
|
||||
"azure-mistral",
|
||||
Validation(options={**NO_RETRIES, "azure_ad_token_provider": raising_token_provider}, api_key=None),
|
||||
),
|
||||
}
|
||||
|
||||
VALIDATION_ROWS: Final = {
|
||||
**{
|
||||
f"{name}-{provider}": (PROVIDERS[provider], validation)
|
||||
for name, validation in GENERIC_VALIDATIONS.items()
|
||||
for provider in PROVIDERS
|
||||
},
|
||||
**{name: (PROVIDERS[provider], validation) for name, (provider, validation) in SPECIFIC_VALIDATIONS.items()},
|
||||
"invalid-provider": (Provider("not_a_provider/ocr-model", OCR_DOCUMENT, {}), Validation()),
|
||||
}
|
||||
|
||||
CREDENTIAL_VARIABLES: Final = ("AZURE_DOCUMENT_INTELLIGENCE_API_KEY", "REDUCTO_API_KEY", "MISTRAL_API_KEY")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
@pytest.mark.parametrize("row", VALIDATION_ROWS.values(), ids=VALIDATION_ROWS.keys())
|
||||
async def test_validation_parity(
|
||||
recording_server: RecordingServer,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
isolated_azure_auth: None,
|
||||
tmp_path: Path,
|
||||
row: tuple[Provider, Validation],
|
||||
asynchronous: bool,
|
||||
) -> None:
|
||||
provider, validation = row
|
||||
for variable in CREDENTIAL_VARIABLES:
|
||||
monkeypatch.delenv(variable, raising=False)
|
||||
recording_server.expected_requests = None
|
||||
|
||||
def invoke() -> Awaitable[OCRResponse]:
|
||||
document: Final = provider.document if validation.document is None else validation.document(tmp_path)
|
||||
options: Final = {**validation.options, "document": dict(document), "api_key": validation.api_key}
|
||||
return call(recording_server, asynchronous, provider, options)
|
||||
|
||||
python: Final = await run_on(False, monkeypatch, invoke)
|
||||
python_requests: Final = len(recording_server.requests)
|
||||
rust: Final = await run_on(True, monkeypatch, invoke)
|
||||
assert len(recording_server.requests) - python_requests == python_requests
|
||||
assert_parity(python, rust, upstream_divergences(asynchronous, provider, validation.options, python))
|
||||
|
|
@ -24,13 +24,6 @@ from tests.test_litellm_rust.support.requests import (
|
|||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
||||
|
||||
@pytest.fixture(params=[False, True], ids=["python", "rust"])
|
||||
def ocr_backend(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> bool:
|
||||
enabled: Final = bool(request.param)
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if enabled else "0")
|
||||
return enabled
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
|
||||
async def test_ocr_contract_upstream_status(
|
||||
|
|
|
|||
98
tests/test_litellm_rust/ocr/test_router_and_proxy.py
Normal file
98
tests/test_litellm_rust/ocr/test_router_and_proxy.py
Normal file
|
|
@ -0,0 +1,98 @@
|
|||
from collections.abc import Mapping
|
||||
from dataclasses import dataclass
|
||||
from types import MappingProxyType
|
||||
from typing import Final
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from litellm import Router
|
||||
from litellm.proxy import proxy_server
|
||||
from tests.test_litellm_rust.support.parity import assert_parity
|
||||
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
|
||||
from tests.test_litellm_rust.support.requests import OCR_DOCUMENT, OCR_MODEL, OCR_RESPONSE
|
||||
|
||||
pytestmark = pytest.mark.requires_rust_extension
|
||||
|
||||
MODEL_GROUP: Final = "ocr"
|
||||
ROUTER_RETRIES: Final = 1
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Row:
|
||||
upstream: ResponseSpec
|
||||
document: Mapping[str, object] = MappingProxyType(dict(OCR_DOCUMENT))
|
||||
deployment: Mapping[str, object] = MappingProxyType({})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class ProxyOutcome:
|
||||
status: int
|
||||
error_type: object
|
||||
error_code: object
|
||||
upstream_requests: int
|
||||
cooled_down: int
|
||||
timeout_header: bool
|
||||
markdown: object
|
||||
|
||||
|
||||
ROWS: Final = {
|
||||
"success": Row(ResponseSpec(body=OCR_RESPONSE)),
|
||||
"unauthorized": Row(ResponseSpec(body={"message": "rejected"}, status=401)),
|
||||
"not-found": Row(ResponseSpec(body={"message": "no such model"}, status=404)),
|
||||
"rate-limited": Row(ResponseSpec(body={"message": "slow down"}, status=429, headers={"retry-after": "0"})),
|
||||
"provider-failure": Row(ResponseSpec(body={"message": "provider unavailable"}, status=500)),
|
||||
"read-timeout": Row(ResponseSpec(body=OCR_RESPONSE, delay=1.5), deployment=MappingProxyType({"timeout": 0.5})),
|
||||
"not-json": Row(ResponseSpec(body=None, raw=b"<html>bad gateway</html>")),
|
||||
"missing-document-url": Row(
|
||||
ResponseSpec(body=OCR_RESPONSE), document=MappingProxyType({"type": "document_url", "document_url": ""})
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def observe(server: RecordingServer, row: Row, monkeypatch: pytest.MonkeyPatch, backend: bool) -> ProxyOutcome:
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if backend else "0")
|
||||
server.default_response = row.upstream
|
||||
before: Final = len(server.requests)
|
||||
router: Final = Router(
|
||||
model_list=[
|
||||
{
|
||||
"model_name": MODEL_GROUP,
|
||||
"litellm_params": {
|
||||
"model": OCR_MODEL,
|
||||
"api_key": "test-key",
|
||||
"api_base": server.base_url,
|
||||
**row.deployment,
|
||||
},
|
||||
}
|
||||
],
|
||||
num_retries=ROUTER_RETRIES,
|
||||
retry_after=0,
|
||||
)
|
||||
monkeypatch.setattr(proxy_server, "llm_router", router)
|
||||
monkeypatch.setattr(proxy_server, "master_key", None)
|
||||
monkeypatch.setattr(proxy_server, "general_settings", {})
|
||||
with TestClient(proxy_server.app, raise_server_exceptions=False) as client:
|
||||
response: Final = client.post("/v1/ocr", json={"model": MODEL_GROUP, "document": dict(row.document)})
|
||||
body: Final = response.json()
|
||||
error: Final = body.get("error") if isinstance(body, dict) else None
|
||||
pages: Final = body.get("pages") if isinstance(body, dict) else None
|
||||
if response.status_code == 200:
|
||||
assert response.headers.get("x-litellm-rust") == ("true" if backend else None)
|
||||
return ProxyOutcome(
|
||||
status=response.status_code,
|
||||
error_type=None if not isinstance(error, dict) else error.get("type"),
|
||||
error_code=None if not isinstance(error, dict) else error.get("code"),
|
||||
upstream_requests=len(server.requests) - before,
|
||||
cooled_down=len(router.cooldown_cache.get_active_cooldowns(router.get_model_ids(), parent_otel_span=None)),
|
||||
timeout_header="x-litellm-timeout" in response.headers,
|
||||
markdown=pages[0].get("markdown") if isinstance(pages, list) and pages else None,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("row", ROWS.values(), ids=ROWS.keys())
|
||||
def test_router_and_proxy_parity(recording_server: RecordingServer, monkeypatch: pytest.MonkeyPatch, row: Row) -> None:
|
||||
recording_server.expected_requests = None
|
||||
python: Final = observe(recording_server, row, monkeypatch, backend=False)
|
||||
rust: Final = observe(recording_server, row, monkeypatch, backend=True)
|
||||
assert_parity(python, rust)
|
||||
142
tests/test_litellm_rust/support/parity.py
Normal file
142
tests/test_litellm_rust/support/parity.py
Normal file
|
|
@ -0,0 +1,142 @@
|
|||
from collections.abc import Awaitable, Callable, Mapping
|
||||
from dataclasses import dataclass, fields, replace
|
||||
from types import MappingProxyType
|
||||
from typing import TYPE_CHECKING, Final, TypeVar
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from litellm.llms.base_llm.ocr.transformation import OCRResponse
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from _typeshed import DataclassInstance
|
||||
|
||||
VOLATILE_HEADERS: Final = frozenset({"date", "server"})
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Outcome:
|
||||
kind: str
|
||||
error_class: str | None = None
|
||||
text: str | None = None
|
||||
message: str | None = None
|
||||
status_code: object = None
|
||||
llm_provider: object = None
|
||||
model: object = None
|
||||
litellm_debug_info: object = None
|
||||
num_retries: object = None
|
||||
max_retries: object = None
|
||||
timeout: object = None
|
||||
response_status: int | None = None
|
||||
response_text: str | None = None
|
||||
response_headers: Mapping[str, str] | None = None
|
||||
litellm_response_headers: Mapping[str, str] | None = None
|
||||
result: object = None
|
||||
|
||||
|
||||
def _headers(value: object) -> Mapping[str, str] | None:
|
||||
if not isinstance(value, httpx.Headers | Mapping):
|
||||
return None
|
||||
return MappingProxyType(
|
||||
{str(name).lower(): str(item) for name, item in value.items() if str(name).lower() not in VOLATILE_HEADERS}
|
||||
)
|
||||
|
||||
|
||||
def _response(error: BaseException) -> httpx.Response | None:
|
||||
try:
|
||||
response: Final = getattr(error, "response", None)
|
||||
except RuntimeError:
|
||||
return None
|
||||
return response if isinstance(response, httpx.Response) else None
|
||||
|
||||
|
||||
def failure_outcome(error: BaseException) -> Outcome:
|
||||
response: Final = _response(error)
|
||||
return Outcome(
|
||||
kind="error",
|
||||
error_class=f"{type(error).__module__}.{type(error).__qualname__}",
|
||||
text=str(error),
|
||||
message=getattr(error, "message", None),
|
||||
status_code=getattr(error, "status_code", None),
|
||||
llm_provider=getattr(error, "llm_provider", None),
|
||||
model=getattr(error, "model", None),
|
||||
litellm_debug_info=getattr(error, "litellm_debug_info", None),
|
||||
num_retries=getattr(error, "num_retries", None),
|
||||
max_retries=getattr(error, "max_retries", None),
|
||||
timeout=getattr(error, "timeout", None),
|
||||
response_status=None if response is None else response.status_code,
|
||||
response_text=None if response is None else response.text,
|
||||
response_headers=None if response is None else _headers(response.headers),
|
||||
litellm_response_headers=_headers(getattr(error, "litellm_response_headers", None)),
|
||||
)
|
||||
|
||||
|
||||
def success_outcome(response: OCRResponse) -> Outcome:
|
||||
return Outcome(kind="ok", result=response.model_dump())
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Divergence:
|
||||
"""A field that differs by design. With `normalize`, both sides are compared after it
|
||||
instead of the field being dropped."""
|
||||
|
||||
field: str
|
||||
reason: str
|
||||
normalize: Callable[[object], object] | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class Masked:
|
||||
"""A field that holds a run-dependent value on both sides, compared after `mask`."""
|
||||
|
||||
field: str
|
||||
reason: str
|
||||
mask: Callable[[object], object]
|
||||
|
||||
|
||||
async def observe(call: Callable[[], Awaitable[OCRResponse]]) -> Outcome:
|
||||
try:
|
||||
return success_outcome(await call())
|
||||
except Exception as error:
|
||||
return failure_outcome(error)
|
||||
|
||||
|
||||
async def run_on(backend: bool, monkeypatch: pytest.MonkeyPatch, call: Callable[[], Awaitable[OCRResponse]]) -> Outcome:
|
||||
monkeypatch.setenv("LITELLM_RUST", "1" if backend else "0")
|
||||
return await observe(call)
|
||||
|
||||
|
||||
ObservedT = TypeVar("ObservedT", bound="DataclassInstance")
|
||||
|
||||
|
||||
def without(outcome: ObservedT, divergences: tuple[Divergence, ...], masks: tuple[Masked, ...] = ()) -> ObservedT:
|
||||
compared: Final = {
|
||||
**{
|
||||
divergence.field: None
|
||||
if divergence.normalize is None
|
||||
else divergence.normalize(getattr(outcome, divergence.field))
|
||||
for divergence in divergences
|
||||
},
|
||||
**{masked.field: masked.mask(getattr(outcome, masked.field)) for masked in masks},
|
||||
}
|
||||
return replace(outcome, **compared)
|
||||
|
||||
|
||||
def assert_parity(
|
||||
python: ObservedT, rust: ObservedT, divergences: tuple[Divergence, ...] = (), masks: tuple[Masked, ...] = ()
|
||||
) -> None:
|
||||
known: Final = frozenset(divergence.field for divergence in divergences) | {masked.field for masked in masks}
|
||||
unknown: Final = known - {field.name for field in fields(python)}
|
||||
assert not unknown, f"divergence registry names unknown fields: {sorted(unknown)}"
|
||||
for divergence in divergences:
|
||||
assert getattr(python, divergence.field) != getattr(rust, divergence.field), (
|
||||
f"registered divergence on {divergence.field!r} no longer diverges; remove it: {divergence.reason}"
|
||||
)
|
||||
expected: Final = without(python, divergences, masks)
|
||||
actual: Final = without(rust, divergences, masks)
|
||||
differences: Final = {
|
||||
field.name: {"python": getattr(expected, field.name), "rust": getattr(actual, field.name)}
|
||||
for field in fields(python)
|
||||
if getattr(expected, field.name) != getattr(actual, field.name)
|
||||
}
|
||||
assert not differences, differences
|
||||
|
|
@ -26,8 +26,11 @@ class ResponseSpec:
|
|||
headers: dict[str, str] = field(default_factory=dict)
|
||||
delay: float = 0
|
||||
events: tuple[tuple[str, object], ...] = ()
|
||||
raw: bytes | None = None
|
||||
|
||||
def payloads(self) -> tuple[bytes, ...]:
|
||||
if self.raw is not None:
|
||||
return (self.raw,)
|
||||
if not self.events:
|
||||
return (json.dumps(self.body).encode(),)
|
||||
return tuple(f"event: {event}\ndata: {json.dumps(data)}\n\n".encode() for event, data in self.events)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue