mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-06 02:48:13 +00:00
4d6fc36fa0
The rust OCR smoke lived under tests/e2e/gateway and spoke raw httpx with ad-hoc dataclasses, diverging from the rest of tests/e2e. Move it to tests/e2e/llm_translation and rebuild it on the shared harness: typed pydantic bodies in models.py (OcrDocument/OcrBody/OcrPage/OcrResponse), a Gateway.ocr() route through the shared transport, Result/unwrap for outcomes, the e2e marker, and the client/scoped_key fixtures. No test touches httpx or requests directly now. Behavior preserved: the config-presence check still reads gateway/litellm-config.yml without a proxy, /model/info confirms the proxy loaded every rust-ocr deployment, and each provider case asserts a well-formed OCR document over /v1/ocr. Also add tests/e2e/CONTRIBUTING.md documenting the end-to-end testing flow so new features land with coverage that walks the feature like production does.
117 lines
4.3 KiB
Python
117 lines
4.3 KiB
Python
"""Live e2e: Rust-backed OCR is reachable through the gateway across providers.
|
|
|
|
The gateway config declares one rust-ocr deployment per provider (mistral,
|
|
azure_ai, azure document intelligence, vertex mistral, vertex deepseek). Start the
|
|
proxy with the Rust OCR path enabled:
|
|
|
|
LITELLM_USE_RUST_OCR=1 litellm --config tests/e2e/gateway/litellm-config.yml
|
|
|
|
Three behaviors are checked: the config declares every provider's deployment (a
|
|
pure config read, no proxy needed); the running proxy loaded them onto /model/info;
|
|
and each one returns a well-formed OCR document over /v1/ocr. Per the e2e
|
|
"skip on environment, fail on behavior" rule, the proxy-backed cases skip when no
|
|
proxy answers but fail (never skip) once a request reaches it, so a provider whose
|
|
credentials are missing surfaces as a hard failure rather than silent green.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
import yaml
|
|
from pydantic import BaseModel
|
|
|
|
from e2e_http import unwrap
|
|
from models import OcrBody, OcrDocument, OcrResponse
|
|
from passthrough_client import PassthroughClient
|
|
|
|
# Tiny in-repo fixtures served via jsdelivr (sha-pinned, immutable) so the request
|
|
# bodies stay stable across runs.
|
|
TEST_PDF_URL = (
|
|
"https://cdn.jsdelivr.net/gh/BerriAI/litellm"
|
|
"@d769e81c90d453240c61fc572cdb27fae06a89d0"
|
|
"/tests/llm_translation/fixtures/dummy.pdf"
|
|
)
|
|
TEST_IMAGE_URL = (
|
|
"https://cdn.jsdelivr.net/gh/BerriAI/litellm"
|
|
"@d769e81c90d453240c61fc572cdb27fae06a89d0"
|
|
"/tests/image_gen_tests/test_image.png"
|
|
)
|
|
|
|
CONFIG_PATH = Path(__file__).resolve().parents[1] / "gateway" / "litellm-config.yml"
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class _OcrCase:
|
|
model: str
|
|
document: OcrDocument
|
|
|
|
|
|
RUST_OCR_CASES: tuple[_OcrCase, ...] = (
|
|
_OcrCase(
|
|
"rust-ocr-mistral",
|
|
OcrDocument(type="document_url", document_url=TEST_PDF_URL),
|
|
),
|
|
_OcrCase(
|
|
"rust-ocr-azure-ai",
|
|
OcrDocument(type="document_url", document_url=TEST_PDF_URL),
|
|
),
|
|
_OcrCase(
|
|
"rust-ocr-azure-document-intelligence",
|
|
OcrDocument(type="document_url", document_url=TEST_PDF_URL),
|
|
),
|
|
_OcrCase(
|
|
"rust-ocr-vertex-mistral",
|
|
OcrDocument(type="document_url", document_url=TEST_PDF_URL),
|
|
),
|
|
_OcrCase(
|
|
"rust-ocr-vertex-deepseek",
|
|
OcrDocument(type="image_url", image_url=TEST_IMAGE_URL),
|
|
),
|
|
)
|
|
|
|
_EXPECTED_MODELS = frozenset(case.model for case in RUST_OCR_CASES)
|
|
_CASE_IDS = tuple(case.model.removeprefix("rust-ocr-") for case in RUST_OCR_CASES)
|
|
|
|
|
|
class _ConfiguredModel(BaseModel):
|
|
model_name: str
|
|
|
|
|
|
class _GatewayConfig(BaseModel):
|
|
model_list: list[_ConfiguredModel]
|
|
|
|
|
|
def _configured_model_names() -> frozenset[str]:
|
|
config = _GatewayConfig.model_validate(yaml.safe_load(CONFIG_PATH.read_text()))
|
|
return frozenset(entry.model_name for entry in config.model_list)
|
|
|
|
|
|
def _assert_ocr_document(response: OcrResponse) -> None:
|
|
assert response.object == "ocr", f"expected object='ocr', got {response.object!r}"
|
|
assert response.model, "response missing the resolved model name"
|
|
assert response.pages, "OCR returned no pages"
|
|
assert response.pages[0].markdown is not None, "first page has no markdown"
|
|
|
|
|
|
def test_rust_ocr_models_declared_in_gateway_config() -> None:
|
|
"""Pure config read (no proxy): every provider's rust-ocr deployment the suite
|
|
exercises is declared in the gateway config the proxy runs with. A case added
|
|
here without a matching deployment fails before any live call is attempted."""
|
|
missing = _EXPECTED_MODELS - _configured_model_names()
|
|
assert not missing, f"rust-ocr models absent from {CONFIG_PATH.name}: {missing}"
|
|
|
|
|
|
@pytest.mark.e2e
|
|
class TestRustOcrGateway:
|
|
def test_gateway_loaded_rust_ocr_models(self, client: PassthroughClient) -> None:
|
|
loaded = frozenset(entry.model_name for entry in client.gateway.model_info())
|
|
missing = _EXPECTED_MODELS - loaded
|
|
assert not missing, f"proxy did not load rust-ocr models: {missing}"
|
|
|
|
@pytest.mark.parametrize("case", RUST_OCR_CASES, ids=_CASE_IDS)
|
|
def test_rust_ocr_response(self, client: PassthroughClient, scoped_key: str, case: _OcrCase) -> None:
|
|
response = unwrap(client.gateway.ocr(scoped_key, OcrBody(model=case.model, document=case.document)))
|
|
_assert_ocr_document(response)
|