diff --git a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs index eabd15677cc..c20a3984edc 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/ocr/transformation.rs @@ -14,7 +14,7 @@ const AZURE_DOCUMENT_INTELLIGENCE_ENDPOINT_ENV: &str = "AZURE_DOCUMENT_INTELLIGE const AZURE_DOCUMENT_INTELLIGENCE_API_VERSION: &str = "2024-11-30"; const AZURE_DOCUMENT_INTELLIGENCE_DEFAULT_DPI: i64 = 96; -const AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS: &[&str] = &["pages"]; +const AZURE_DOCUMENT_INTELLIGENCE_SUPPORTED_OCR_PARAMS: &[&str] = &["pages", "features"]; pub struct AzureAiOcrConfig; pub struct AzureDocumentIntelligenceOcrConfig; @@ -192,6 +192,46 @@ fn normalize_pages_param(pages: &Value) -> CoreResult> { } } +fn feature_token_is_valid(token: &str) -> bool { + let Some((first, rest)) = token.as_bytes().split_first() else { + return false; + }; + first.is_ascii_alphabetic() && rest.iter().all(u8::is_ascii_alphanumeric) +} + +fn invalid_features_error(features: &Value) -> CoreError { + CoreError::InvalidRequest(format!( + "Invalid `features` for Azure Document Intelligence: {features:?}. Expected a list of feature names or a comma-separated string like 'keyValuePairs' or 'keyValuePairs,languages'." + )) +} + +fn normalize_features_param(features: &Value) -> CoreResult> { + let normalized = match features { + Value::String(value) => value + .split(',') + .map(str::trim) + .collect::>() + .join(","), + Value::Array(values) if values.is_empty() => return Ok(None), + Value::Array(values) => values + .iter() + .map(Value::as_str) + .collect::>>() + .ok_or_else(|| invalid_features_error(features))? + .into_iter() + .map(str::trim) + .collect::>() + .join(","), + _ => return Err(invalid_features_error(features)), + }; + + if normalized.split(',').all(feature_token_is_valid) { + Ok(Some(normalized)) + } else { + Err(invalid_features_error(features)) + } +} + pub fn complete_document_intelligence_url( api_base: Option<&str>, model: &str, @@ -213,6 +253,13 @@ pub fn complete_document_intelligence_url( url.push_str(&normalized); } + if let Some(features) = optional_params.get("features") + && let Some(normalized) = normalize_features_param(features)? + { + url.push_str("&features="); + url.push_str(&normalized); + } + Ok(url) } @@ -475,6 +522,103 @@ mod tests { ); } + #[test] + fn document_intelligence_url_normalizes_features() { + let params = serde_json::Map::from_iter([( + "features".to_string(), + json!("keyValuePairs, languages"), + )]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&features=keyValuePairs,languages" + ); + } + + #[test] + fn document_intelligence_url_combines_pages_and_feature_list() { + let params = serde_json::Map::from_iter([ + ("pages".to_string(), json!([0, 1, 2])), + ( + "features".to_string(), + json!([" keyValuePairs ", "languages"]), + ), + ]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30&pages=1,2,3&features=keyValuePairs,languages" + ); + } + + #[test] + fn document_intelligence_url_omits_empty_feature_list() { + let params = serde_json::Map::from_iter([("features".to_string(), json!([]))]); + let url = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect("url builds"); + + assert_eq!( + url, + "https://example.cognitiveservices.azure.com/documentintelligence/documentModels/prebuilt-layout:analyze?api-version=2024-11-30" + ); + } + + #[test] + fn document_intelligence_url_rejects_invalid_features() { + for features in [ + json!("keyValuePairs&pages=9"), + json!(""), + json!(["keyValuePairs", 1]), + json!({"feature": "keyValuePairs"}), + ] { + let params = serde_json::Map::from_iter([("features".to_string(), features.clone())]); + let error = complete_document_intelligence_url( + Some("https://example.cognitiveservices.azure.com"), + "prebuilt-layout", + ¶ms, + &|_| None, + ) + .expect_err("invalid features must fail"); + + assert!( + matches!(error, CoreError::InvalidRequest(message) if message.contains("Invalid `features`")), + "features={features:?}" + ); + } + } + + #[test] + fn document_intelligence_maps_features() { + let params = Map::from_iter([ + ("features".to_string(), json!(["keyValuePairs"])), + ("unsupported".to_string(), json!(true)), + ]); + + assert_eq!( + AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG.map_ocr_params(¶ms), + Map::from_iter([("features".to_string(), json!(["keyValuePairs"]))]) + ); + } + #[test] fn document_intelligence_request_uses_base64_source_for_data_uri() { let body = AZURE_DOCUMENT_INTELLIGENCE_OCR_CONFIG diff --git a/pyproject.toml b/pyproject.toml index 576f18c3597..4244657b17a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -175,6 +175,7 @@ litellm-proxy = "litellm.proxy.client.cli:cli" dev = [ "diff-cover==9.7.2", "hypothesis==6.165.10", + "reportlab==5.0.1", "basedpyright==1.39.7", "keyring==25.7.0", "pytest==9.0.3", diff --git a/tests/route_parity/fixtures/media.py b/tests/route_parity/fixtures/media.py new file mode 100644 index 00000000000..2bb3cd99601 --- /dev/null +++ b/tests/route_parity/fixtures/media.py @@ -0,0 +1,225 @@ +from __future__ import annotations + +import base64 +from functools import cache +from io import BytesIO +from typing import Final +from urllib.parse import quote + +from PIL import Image, ImageDraw +from reportlab.graphics.barcode import code128 # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib import colors # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib.pagesizes import letter # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.lib.utils import ImageReader # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs +from reportlab.pdfgen import canvas # pyright: ignore[reportMissingTypeStubs] # ReportLab has no stubs + + +def dummy_image_url(text: str, font_size: int, width: int = 800, height: int = 300) -> str: + return f"https://dummyjson.com/image/{width}x{height}/ffffff/000000?text={quote(text)}&fontSize={font_size}" + + +_GLYPHS: Final = { + "O": ("01110", "10001", "10001", "10001", "10001", "10001", "01110"), + "C": ("01111", "10000", "10000", "10000", "10000", "10000", "01111"), + "R": ("11110", "10001", "10001", "11110", "10100", "10010", "10001"), + "1": ("00100", "01100", "00100", "00100", "00100", "00100", "01110"), + "2": ("01110", "10001", "00001", "00010", "00100", "01000", "11111"), + "3": ("11110", "00001", "00001", "01110", "00001", "00001", "11110"), +} + + +@cache +def structured_image_bytes() -> bytes: + image: Final = Image.new("RGB", (320, 80), "white") + draw: Final = ImageDraw.Draw(image) + scale: Final = 8 + cursor_x = 24 + for character in "OCR 123": + if character == " ": + cursor_x += scale * 3 + continue + for glyph_y, row in enumerate(_GLYPHS[character]): + for glyph_x, filled in enumerate(row): + if filled == "1": + x = cursor_x + glyph_x * scale + y = 12 + glyph_y * scale + draw.rectangle((x, y, x + scale - 1, y + scale - 1), fill="black") + cursor_x += scale * 6 + output: Final = BytesIO() + image.save(output, format="PNG") + return output.getvalue() + + +@cache +def structured_image_data_uri() -> str: + encoded: Final = base64.b64encode(structured_image_bytes()).decode("ascii") + return f"data:image/png;base64,{encoded}" + + +def _draw_header(pdf: canvas.Canvas, title: str, page_number: int) -> None: + pdf.setFillColor(colors.black) + pdf.setFont("Helvetica", 11) + pdf.drawString(45, 770, "Quarterly Operations Report") + pdf.setFont("Helvetica-Bold", 16) + pdf.drawString(45, 745, title) + pdf.setFont("Helvetica", 9) + pdf.drawString(45, 30, f"Confidential | Page {page_number} of 5") + + +def _draw_body(pdf: canvas.Canvas, page_number: int) -> None: + pdf.setFont("Helvetica", 10) + for line_number in range(1, 9): + pdf.drawString( + 45, + 500 - (line_number * 28), + f"Section {page_number}.{line_number}: Invoice totals, regional revenue, and reconciliation notes.", + ) + + +def _diagram_image(width: int, height: int, accent: tuple[int, int, int]) -> Image.Image: + image: Final = Image.new("RGB", (width, height), (242, 246, 252)) + draw: Final = ImageDraw.Draw(image) + for coordinate in range(0, max(width, height), 40): + draw.line((coordinate, 0, coordinate, height), fill=(32, 32, 32), width=3) + draw.line((0, coordinate, width, coordinate), fill=(32, 32, 32), width=3) + draw.line((0, 0, width, height), fill=accent, width=8) + draw.line((width, 0, 0, height), fill=accent, width=8) + draw.rectangle((width // 4, height // 4, width * 3 // 4, height * 3 // 4), outline=accent, width=6) + return image + + +def _draw_embedded_images(pdf: canvas.Canvas) -> None: + images: Final = ( + (_diagram_image(320, 320, (51, 115, 217)), 455, 655, 70, 70), + (_diagram_image(360, 320, (38, 151, 92)), 455, 565, 70, 62), + (_diagram_image(120, 120, (219, 68, 55)), 455, 500, 45, 45), + ) + for image, x, y, width, height in images: + pdf.drawImage( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + ImageReader(image), x, y, width=width, height=height, mask="auto" + ) + + +def _draw_table_page(pdf: canvas.Canvas) -> None: + columns: Final = (45, 245, 405, 565) + tables: Final = ( + ( + (730, 695, 660, 625), + ( + ("Item", "Quantity", "Amount", 707), + ("Document analysis", "2", "120.00", 672), + ("OCR verification", "1", "80.00", 637), + ), + ), + ( + (600, 565, 530, 495), + ( + ("Item continued", "Quantity", "Amount", 577), + ("Fixture validation", "3", "45.00", 542), + ("Provider review", "1", "25.00", 507), + ), + ), + ) + for rows, values in tables: + for x in columns: + pdf.line(x, rows[-1], x, rows[0]) + for y in rows: + pdf.line(45, y, 565, y) + for item, quantity, amount, y in values: + pdf.drawString(55, y, item) + pdf.drawString(255, y, quantity) + pdf.drawString(415, y, amount) + + +def _draw_chart_page(pdf: canvas.Canvas) -> None: + bars: Final = ((70, 70), (170, 115), (270, 90), (370, 130)) + pdf.setFillColor(colors.HexColor("#3373D9")) + for x, height in bars: + pdf.rect(x, 610, 65, height, fill=1, stroke=0) + pdf.setFillColor(colors.black) + for quarter, x in zip(("Q1", "Q2", "Q3", "Q4"), (90, 190, 290, 390), strict=True): + pdf.drawString(x, 590, quarter) + pdf.drawString(45, 550, "Formula: gross margin = (revenue - cost) / revenue") + _draw_embedded_images(pdf) + + +def _draw_metadata_page(pdf: canvas.Canvas) -> None: + pdf.setFont("Helvetica", 12) + pdf.drawString(45, 700, "Invoice Number: INV-2048") + pdf.drawString(45, 675, "Purchase Order: PO-4096") + pdf.setFillColor(colors.HexColor("#F2E65A")) + pdf.rect(40, 555, 500, 24, fill=1, stroke=0) + pdf.setFillColor(colors.black) + pdf.drawString(45, 560, "Highlighted total requiring review") + pdf.drawString(45, 530, "Reviewer comment: verify the highlighted total before approval") + pdf.setFillColor(colors.red) + pdf.drawString(45, 495, "Revised total: 245.00") + pdf.line(45, 501, 150, 501) + pdf.setFillColor(colors.black) + pdf.linkURL( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "https://example.com/invoices/INV-2048", (45, 575, 300, 590), relative=0 + ) + pdf.highlightAnnotation( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "Total highlighted for review", + Rect=(40, 555, 540, 579), + QuadPoints=(40, 579, 540, 579, 40, 555, 540, 555), + ) + pdf.textAnnotation( # pyright: ignore[reportUnknownMemberType] # ReportLab has no stubs + "Verify the highlighted total", Rect=(520, 525, 540, 545) + ) + pdf.drawString(45, 575, "https://example.com/invoices/INV-2048") + barcode: Final = code128.Code128("5901234123457", barHeight=70, barWidth=1.2) + barcode.drawOn(pdf, 90, 130) + + +def _draw_signature_page(pdf: canvas.Canvas) -> None: + pdf.saveState() + pdf.setFillColor(colors.lightgrey) + pdf.setFont("Helvetica-Bold", 54) + pdf.translate(110, 390) + pdf.rotate(25) + pdf.drawString(0, 0, "DRAFT") + pdf.restoreState() + pdf.setFillColor(colors.black) + pdf.setFont("Helvetica", 12) + pdf.drawString(45, 635, "Approved by: Jordan Lee") + pdf.line(45, 610, 310, 610) + pdf.bezier(55, 595, 75, 625, 112, 602, 155, 600) + pdf.drawString(45, 580, "Signature") + + +def _draw_appendix_page(pdf: canvas.Canvas) -> None: + pdf.setFont("Helvetica-Bold", 14) + pdf.drawString(45, 700, "1. Scope") + pdf.drawString(45, 650, "2. Findings") + pdf.drawString(45, 600, "3. Recommendations") + + +@cache +def structured_pdf_bytes() -> bytes: + output: Final = BytesIO() + pdf: Final = canvas.Canvas(output, pagesize=letter, pageCompression=0, invariant=1) + pdf.setTitle("Quarterly Operations Report") + pdf.setAuthor("LiteLLM OCR fixture generator") + pdf.setSubject("Semantic OCR coverage for tables, figures, annotations, and metadata") + pdf.setKeywords("OCR, invoice, table, figure, annotation") + pages: Final = ( + ("Invoice Summary and Line Items", _draw_table_page), + ("Revenue Chart and Formula Review", _draw_chart_page), + ("Key Values, Link, Highlight, and Comment", _draw_metadata_page), + ("Approval Signature and Watermark", _draw_signature_page), + ("Appendix with Section Boundaries", _draw_appendix_page), + ) + for page_number, (title, draw_page) in enumerate(pages, start=1): + _draw_header(pdf, title, page_number) + draw_page(pdf) + _draw_body(pdf, page_number) + pdf.showPage() + pdf.save() + return output.getvalue() + + +@cache +def structured_pdf_data_uri() -> str: + encoded: Final = base64.b64encode(structured_pdf_bytes()).decode("ascii") + return f"data:application/pdf;base64,{encoded}" diff --git a/tests/route_parity/fixtures/test_media.py b/tests/route_parity/fixtures/test_media.py new file mode 100644 index 00000000000..5944074bf11 --- /dev/null +++ b/tests/route_parity/fixtures/test_media.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +import base64 +from io import BytesIO +from typing import Final, cast + +from PIL import Image + +from tests.route_parity.fixtures.media import dummy_image_url, structured_image_bytes, structured_image_data_uri + + +def test_dummy_image_url_encodes_text_and_dimensions() -> None: + assert dummy_image_url("invoice 123", 24, width=320, height=80) == ( + "https://dummyjson.com/image/320x80/ffffff/000000?text=invoice%20123&fontSize=24" + ) + + +def test_structured_image_is_local_content_bearing_png() -> None: + png: Final = structured_image_bytes() + encoded: Final = structured_image_data_uri().partition(",")[2] + image: Final = Image.open(BytesIO(png)) + colors: Final = cast(list[tuple[int, tuple[int, int, int]]], image.getcolors(maxcolors=2)) + + assert png.startswith(b"\x89PNG\r\n\x1a\n") + assert base64.b64decode(encoded, validate=True) == png + assert image.size == (320, 80) + assert {color for _, color in colors} == {(0, 0, 0), (255, 255, 255)} diff --git a/tests/test_litellm/ocr/fixtures/azure.py b/tests/test_litellm/ocr/fixtures/azure.py index 3db682b0baa..bb7f54632a1 100644 --- a/tests/test_litellm/ocr/fixtures/azure.py +++ b/tests/test_litellm/ocr/fixtures/azure.py @@ -5,25 +5,24 @@ from typing import Final, Literal, cast from hypothesis import strategies as st from hypothesis.strategies import SearchStrategy -from pydantic import Field, field_validator +from pydantic import Field, StrictInt, StrictStr, field_validator from tests.route_parity.fixtures.recording import ProviderSpec from tests.test_litellm.ocr.fixtures.base import OcrDocument, OcrSdkInputBase from tests.test_litellm.ocr.fixtures.common import ( OcrFixtureClient, OcrRecordingTarget, + image_document, invoke_with_api_key, parameter_strategy, pdf_document, - public_document_strategy, sampled_list_strategy, + sampled_parameter_group_strategy, sampled_scalar_strategy, ) from tests.test_litellm.ocr.fixtures.mistral import ( - MISTRAL_MODEL, MistralCompatibleOcrSdkInput, - MistralOcrSdkInput, - mistral_input_strategy, + mistral_input_values_strategy, ) AzureMistralModel = Literal["azure_ai/mistral-document-ai-2512",] @@ -39,6 +38,12 @@ AZURE_DOCUMENT_INTELLIGENCE_MODELS: Final[tuple[AzureDocumentIntelligenceModel, "azure_ai/doc-intelligence/prebuilt-layout", "azure_ai/doc-intelligence/prebuilt-document", ) +# API v4 replaces prebuilt-document with prebuilt-layout plus keyValuePairs. Keep +# the broader fixture model above so existing recordings remain loadable. +AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS: Final[tuple[AzureDocumentIntelligenceModel, ...]] = ( + "azure_ai/doc-intelligence/prebuilt-read", + "azure_ai/doc-intelligence/prebuilt-layout", +) class AzureMistralOcrSdkInput(MistralCompatibleOcrSdkInput): @@ -59,26 +64,28 @@ class AzureDocumentIntelligenceOcrSdkInput(OcrSdkInputBase): model: AzureDocumentIntelligenceModel document: OcrDocument custom_llm_provider: Literal["azure_ai"] | None = None - pages: str | list[int] | None = None + pages: str | list[StrictInt] | list[StrictStr] | None = None features: str | list[str] | None = None req_format: Literal["litellm"] = "litellm" -def _as_azure_mistral(case_input: MistralOcrSdkInput, model: AzureMistralModel) -> AzureMistralOcrSdkInput: - values: Final = case_input.model_dump( - mode="python", - exclude={"boundary", "model", "custom_llm_provider"}, - exclude_unset=True, - ) +def _azure_mistral_input(values: dict[str, object], model: AzureMistralModel) -> AzureMistralOcrSdkInput: return AzureMistralOcrSdkInput.model_validate({**values, "model": model}) +def azure_mistral_input_strategy(inline_image_data_uri: str) -> SearchStrategy[AzureMistralOcrSdkInput]: + # Foundry's active gateway schema rejects 2512-only controls and + # document_annotation_prompt, even though native Mistral accepts them. + return st.builds( + _azure_mistral_input, + values=mistral_input_values_strategy("2505", inline_image_data_uri, include_document_annotation_prompt=False), + model=sampled_scalar_strategy(AZURE_MISTRAL_MODELS), + ) + + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL: Final[AzureDocumentIntelligenceModel] = ( "azure_ai/doc-intelligence/prebuilt-layout" ) -_AZURE_DOCUMENT_INTELLIGENCE_DOCUMENT_MODEL: Final[AzureDocumentIntelligenceModel] = ( - "azure_ai/doc-intelligence/prebuilt-document" -) def _document_intelligence_input( @@ -93,26 +100,41 @@ def _document_intelligence_input( def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocumentIntelligenceOcrSdkInput]: document: Final = pdf_document() - pages: Final = st.one_of( - parameter_strategy("pages", sampled_list_strategy(((0,), (0, 1)))), - parameter_strategy("pages", sampled_scalar_strategy(("1", "1,2", "1-2"))), + pages: Final = parameter_strategy( + "pages", + st.one_of( + sampled_list_strategy(((0,), (2, 0, 0, 1))), + sampled_list_strategy((("1", "2-4"),)), + sampled_scalar_strategy(("1-4, 5",)), + ), ) - common_features: Final = parameter_strategy( + features: Final = parameter_strategy( "features", st.one_of( sampled_list_strategy( - (("languages",), ("ocrHighResolution",), ("barcodes",), ("formulas",), ("styleFont",)) + ( + ("languages",), + ("ocrHighResolution",), + ("barcodes",), + ("formulas",), + ("styleFont",), + ("keyValuePairs",), + ) ), - sampled_scalar_strategy(("languages,styleFont",)), + sampled_scalar_strategy(("languages, styleFont",)), ), ) + combined_query: Final = sampled_parameter_group_strategy( + ((("pages", (0, 1)), ("features", ("languages", "styleFont"))),) + ) return st.one_of( - st.sampled_from(AZURE_DOCUMENT_INTELLIGENCE_MODELS).map( + st.sampled_from(AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS).map( lambda model: _document_intelligence_input(model, document) ), - public_document_strategy().map( - lambda selected_document: _document_intelligence_input( - _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, selected_document + st.just( + _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, + image_document("invoice 123", 24), ) ), pages.map( @@ -120,16 +142,14 @@ def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocument _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params ) ), - common_features.map( + features.map( lambda optional_params: _document_intelligence_input( _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params ) ), - st.just( - _document_intelligence_input( - _AZURE_DOCUMENT_INTELLIGENCE_DOCUMENT_MODEL, - document, - {"features": ["keyValuePairs"]}, + combined_query.map( + lambda optional_params: _document_intelligence_input( + _AZURE_DOCUMENT_INTELLIGENCE_CANONICAL_MODEL, document, optional_params ) ), st.just( @@ -143,7 +163,7 @@ def azure_document_intelligence_input_strategy() -> SearchStrategy[AzureDocument def azure_mistral_recording_targets( - environ: Mapping[str, str], client: OcrFixtureClient + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str ) -> tuple[OcrRecordingTarget, ...]: api_key: Final = environ.get("AZURE_AI_API_KEY") upstream_base: Final = environ.get("AZURE_AI_API_BASE") @@ -155,11 +175,7 @@ def azure_mistral_recording_targets( provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), strategy=cast( SearchStrategy[OcrSdkInputBase], - st.sampled_from(AZURE_MISTRAL_MODELS).flatmap( - lambda model: mistral_input_strategy(MISTRAL_MODEL, feature_level="2512").map( - lambda case_input: _as_azure_mistral(case_input, model) - ) - ), + azure_mistral_input_strategy(inline_image_data_uri), ), invocation=invoke_with_api_key(client, api_key), ), diff --git a/tests/test_litellm/ocr/fixtures/common.py b/tests/test_litellm/ocr/fixtures/common.py index 9b24586e8a2..d9733891e2c 100644 --- a/tests/test_litellm/ocr/fixtures/common.py +++ b/tests/test_litellm/ocr/fixtures/common.py @@ -1,14 +1,13 @@ from __future__ import annotations -import base64 from dataclasses import dataclass, field -from pathlib import Path -from typing import Final, Protocol, TypeVar -from urllib.parse import quote +from functools import cache +from typing import Final, Literal, Protocol, TypeVar from hypothesis import strategies as st from hypothesis.strategies import SearchStrategy +from tests.route_parity.fixtures.media import dummy_image_url, structured_pdf_data_uri from tests.route_parity.fixtures.pipeline import RecordingTarget from tests.test_litellm.ocr.fixtures.base import ( DocumentUrlDocument, @@ -40,22 +39,46 @@ class ApiKeyOcrInvocation: def image_document(text: str, font_size: int) -> ImageUrlDocument: - url: Final = f"https://dummyjson.com/image/800x300/ffffff/000000?text={quote(text)}&fontSize={font_size}" - return ImageUrlDocument(type="image_url", image_url=url) + return ImageUrlDocument(type="image_url", image_url=dummy_image_url(text, font_size)) -def fixture_pdf_data_uri() -> str: - fixture: Final = Path(__file__).resolve().parents[3] / "llm_translation" / "fixtures" / "dummy.pdf" - encoded: Final = base64.b64encode(fixture.read_bytes()).decode("ascii") - return f"data:application/pdf;base64,{encoded}" +def image_data_document(data_uri: str) -> ImageUrlDocument: + return ImageUrlDocument(type="image_url", image_url=data_uri) +@cache +def remote_pdf_document() -> DocumentUrlDocument: + return DocumentUrlDocument( + type="document_url", + document_url="https://www.w3.org/WAI/ER/tests/xhtml/testfiles/resources/pdf/dummy.pdf", + ) + + +@cache def pdf_document() -> DocumentUrlDocument: - return DocumentUrlDocument(type="document_url", document_url=fixture_pdf_data_uri()) + return DocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri()) -def public_document_strategy() -> SearchStrategy[ImageUrlDocument | DocumentUrlDocument]: - return st.sampled_from((image_document("invoice 123", 24), pdf_document())) +def document_transport_strategy(inline_image_data_uri: str) -> SearchStrategy[ImageUrlDocument | DocumentUrlDocument]: + transports: Final[tuple[Literal["remote_image", "inline_image", "remote_pdf", "inline_pdf"], ...]] = ( + "remote_image", + "inline_image", + "remote_pdf", + "inline_pdf", + ) + + def as_document( + transport: Literal["remote_image", "inline_image", "remote_pdf", "inline_pdf"], + ) -> ImageUrlDocument | DocumentUrlDocument: + if transport == "remote_image": + return image_document("invoice 123", 24) + if transport == "inline_image": + return image_data_document(inline_image_data_uri) + if transport == "remote_pdf": + return remote_pdf_document() + return pdf_document() + + return st.sampled_from(transports).map(as_document) def sampled_scalar_strategy(values: tuple[ValueT, ...]) -> SearchStrategy[ValueT]: diff --git a/tests/test_litellm/ocr/fixtures/mistral.py b/tests/test_litellm/ocr/fixtures/mistral.py index b4dd0171533..cfff9d25bce 100644 --- a/tests/test_litellm/ocr/fixtures/mistral.py +++ b/tests/test_litellm/ocr/fixtures/mistral.py @@ -4,7 +4,7 @@ from collections.abc import Mapping from typing import Final, Literal, cast from hypothesis import strategies as st -from hypothesis.strategies import DrawFn, SearchStrategy +from hypothesis.strategies import SearchStrategy from pydantic import Field, model_validator from typing_extensions import Self @@ -18,10 +18,10 @@ from tests.test_litellm.ocr.fixtures.common import ( OcrFixtureClient, OcrRecordingTarget, annotation_format, - image_document, + document_transport_strategy, invoke_with_api_key, parameter_strategy, - public_document_strategy, + pdf_document, sampled_list_strategy, sampled_parameter_group_strategy, sampled_scalar_strategy, @@ -113,7 +113,14 @@ def _feature_level(model: str) -> MistralFeatureLevel: return "2505" -def mistral_optional_params_strategy(feature_level: MistralFeatureLevel) -> SearchStrategy[dict[str, object]]: +def _optional_param_strategies( + *, + include_document_annotation_prompt: bool = True, +) -> tuple[ + tuple[SearchStrategy[dict[str, object]], ...], + tuple[SearchStrategy[dict[str, object]], ...], + tuple[SearchStrategy[dict[str, object]], ...], +]: annotation: Final = annotation_format("document_title") common: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( parameter_strategy("pages", sampled_list_strategy(((0,), (0, 1)))), @@ -125,16 +132,21 @@ def mistral_optional_params_strategy(feature_level: MistralFeatureLevel) -> Sear sampled_scalar_strategy((annotation_format("bounding_boxes"),)), ), parameter_strategy("document_annotation_format", sampled_scalar_strategy((annotation,))), - sampled_parameter_group_strategy( + *( ( - ( - ("document_annotation_format", annotation), - ("document_annotation_prompt", "Extract the visible title"), + sampled_parameter_group_strategy( + ( + ( + ("document_annotation_format", annotation), + ("document_annotation_prompt", "Extract the visible title"), + ), + ) ), ) + if include_document_annotation_prompt + else () ), parameter_strategy("confidence_scores_granularity", sampled_scalar_strategy(("page", "word"))), - parameter_strategy("id", sampled_scalar_strategy(("case-1",))), ) feature_2512: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( parameter_strategy("extract_header", sampled_scalar_strategy((False, True))), @@ -142,9 +154,21 @@ def mistral_optional_params_strategy(feature_level: MistralFeatureLevel) -> Sear parameter_strategy("table_format", sampled_scalar_strategy(("markdown", "html"))), ) feature_4: Final[tuple[SearchStrategy[dict[str, object]], ...]] = ( + parameter_strategy("pages", sampled_scalar_strategy(("0-2",))), parameter_strategy("include_blocks", sampled_scalar_strategy((False, True))), sampled_parameter_group_strategy(((("include_blocks", True), ("confidence_scores_granularity", "block")),)), ) + return common, feature_2512, feature_4 + + +def mistral_optional_params_strategy( + feature_level: MistralFeatureLevel, + *, + include_document_annotation_prompt: bool = True, +) -> SearchStrategy[dict[str, object]]: + common, feature_2512, feature_4 = _optional_param_strategies( + include_document_annotation_prompt=include_document_annotation_prompt + ) return st.one_of( *common, *(feature_2512 if feature_level in {"2512", "4"} else ()), @@ -152,25 +176,72 @@ def mistral_optional_params_strategy(feature_level: MistralFeatureLevel) -> Sear ) -@st.composite -def mistral_input_strategy( - draw: DrawFn, +def _mistral_input_values( + document: OcrDocument, + optional_params: dict[str, object] | None = None, +) -> dict[str, object]: + return {"document": document, **(optional_params or {})} + + +def _mistral_input( model: str, - feature_level: MistralFeatureLevel | None = None, + document: OcrDocument, + optional_params: dict[str, object] | None = None, ) -> MistralOcrSdkInput: - canonical_document: Final = image_document("invoice 123", 24) - values: Final = draw( - st.one_of( - public_document_strategy().map(lambda document: {"document": document}), - mistral_optional_params_strategy(feature_level or _feature_level(model)).map( - lambda optional_params: {"document": canonical_document, **optional_params} - ), - ) + return MistralOcrSdkInput.model_validate({"model": model, **_mistral_input_values(document, optional_params)}) + + +def mistral_input_values_strategy( + feature_level: MistralFeatureLevel, + inline_image_data_uri: str, + *, + include_document_annotation_prompt: bool = True, +) -> SearchStrategy[dict[str, object]]: + option_document: Final = pdf_document() + return st.one_of( + document_transport_strategy(inline_image_data_uri).map(_mistral_input_values), + mistral_optional_params_strategy( + feature_level, + include_document_annotation_prompt=include_document_annotation_prompt, + ).map(lambda optional_params: _mistral_input_values(option_document, optional_params)), ) - return MistralOcrSdkInput.model_validate({"model": model, **values}) -def mistral_recording_targets(environ: Mapping[str, str], client: OcrFixtureClient) -> tuple[OcrRecordingTarget, ...]: +def mistral_input_strategy( + model: str, + inline_image_data_uri: str, + feature_level: MistralFeatureLevel | None = None, +) -> SearchStrategy[MistralOcrSdkInput]: + return mistral_input_values_strategy(feature_level or _feature_level(model), inline_image_data_uri).map( + lambda values: MistralOcrSdkInput.model_validate({"model": model, **values}) + ) + + +def _mistral_recording_strategy(inline_image_data_uri: str) -> SearchStrategy[MistralOcrSdkInput]: + document: Final = pdf_document() + baseline_models: Final = tuple(model for model in MISTRAL_MODELS if model != MISTRAL_MODEL) + common, feature_2512, feature_4 = _optional_param_strategies() + common_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*common) + feature_2512_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_2512) + feature_4_options: Final[SearchStrategy[dict[str, object]]] = st.one_of(*feature_4) + return st.one_of( + sampled_scalar_strategy(baseline_models).map(lambda model: _mistral_input(model, document)), + document_transport_strategy(inline_image_data_uri).map( + lambda selected_document: _mistral_input(MISTRAL_MODEL, selected_document) + ), + common_options.map(lambda optional_params: _mistral_input(MISTRAL_MODEL, document, optional_params)), + feature_2512_options.map( + lambda optional_params: _mistral_input("mistral/mistral-ocr-2512", document, optional_params) + ), + feature_4_options.map( + lambda optional_params: _mistral_input("mistral/mistral-ocr-4-1", document, optional_params) + ), + ) + + +def mistral_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: api_key: Final = environ.get("MISTRAL_API_KEY") if not api_key: return () @@ -182,7 +253,7 @@ def mistral_recording_targets(environ: Mapping[str, str], client: OcrFixtureClie provider_spec=ProviderSpec(upstream_base=upstream_base), strategy=cast( SearchStrategy[OcrSdkInputBase], - sampled_scalar_strategy(MISTRAL_MODELS).flatmap(mistral_input_strategy), + _mistral_recording_strategy(inline_image_data_uri), ), invocation=invoke_with_api_key(client, api_key), ), diff --git a/tests/test_litellm/ocr/fixtures/record.py b/tests/test_litellm/ocr/fixtures/record.py index 5d6ea9ed6ab..5151ab26d72 100644 --- a/tests/test_litellm/ocr/fixtures/record.py +++ b/tests/test_litellm/ocr/fixtures/record.py @@ -10,6 +10,7 @@ from dotenv import load_dotenv import litellm from litellm.rust_bridge.ocr import use_litellm_rust +from tests.route_parity.fixtures.media import structured_image_data_uri from tests.route_parity.fixtures.pipeline import parse_recording_args, record_fixtures from tests.route_parity.fixtures.store import fixture_directory from tests.test_litellm.ocr.fixtures.azure import ( @@ -37,13 +38,14 @@ class LiteLLMOcrFixtureClient: def discover_targets( environ: Mapping[str, str], client: OcrFixtureClient, + inline_image_data_uri: str, ) -> tuple[OcrRecordingTarget, ...]: return ( - *mistral_recording_targets(environ, client), - *azure_mistral_recording_targets(environ, client), + *mistral_recording_targets(environ, client, inline_image_data_uri), + *azure_mistral_recording_targets(environ, client, inline_image_data_uri), *azure_document_intelligence_recording_targets(environ, client), - *vertex_recording_targets(environ, client), - *reducto_recording_targets(environ, client), + *vertex_recording_targets(environ, client, inline_image_data_uri), + *reducto_recording_targets(environ, client, inline_image_data_uri), ) @@ -58,7 +60,8 @@ def main() -> int: load_dotenv() args: Final = parse_recording_args() client: Final = LiteLLMOcrFixtureClient(cast(OcrSdkCall, litellm.ocr)) - targets: Final = require_targets(discover_targets(os.environ, client)) + inline_image_data_uri: Final = structured_image_data_uri() + targets: Final = require_targets(discover_targets(os.environ, client, inline_image_data_uri)) root: Final = fixture_directory( args.fixture_dir, os.environ.get(FIXTURE_DIR_ENV), diff --git a/tests/test_litellm/ocr/fixtures/reducto.py b/tests/test_litellm/ocr/fixtures/reducto.py index b34870aae42..01babd7f031 100644 --- a/tests/test_litellm/ocr/fixtures/reducto.py +++ b/tests/test_litellm/ocr/fixtures/reducto.py @@ -11,12 +11,13 @@ from pydantic import Field, field_validator, model_validator from typing_extensions import Self from tests.route_parity.fixture_models import FixtureModel, JsonObject +from tests.route_parity.fixtures.media import structured_pdf_data_uri from tests.route_parity.fixtures.recording import ProviderSpec from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase from tests.test_litellm.ocr.fixtures.common import ( OcrFixtureClient, OcrRecordingTarget, - fixture_pdf_data_uri, + image_data_document, invoke_with_api_key, parameter_strategy, sampled_list_strategy, @@ -102,7 +103,7 @@ _REDUCTO_RETURN_IMAGE_GROUPS: Final[tuple[tuple[ReductoReturnImage, ...], ...]] ("figure",), ("table",), ("page",), - ("figure", "table", "page"), + ("figure", "table"), ) @@ -170,14 +171,17 @@ class ReductoHybridVpcSettings(FixtureModel): ReductoPageSelection = ReductoPageRange | list[ReductoPageRange] | list[int] | list[str] ReductoV3Model = Literal["reducto/parse-v3", "parse-v3"] ReductoLegacyModel = Literal["reducto/parse-legacy", "parse-legacy"] +_ReductoV3Route = Literal["qualified", "image", "unqualified"] +_ReductoLegacyRoute = Literal["qualified", "unqualified"] REDUCTO_V3_MODELS: Final[tuple[Literal["reducto/parse-v3"], ...]] = ("reducto/parse-v3",) REDUCTO_LEGACY_MODELS: Final[tuple[Literal["reducto/parse-legacy"], ...]] = ("reducto/parse-legacy",) class ReductoSettings(FixtureModel): + model: Literal["r-1"] | None = None ocr_system: Literal["standard", "legacy"] = "standard" - extraction_mode: Literal["ocr", "hybrid"] = "hybrid" + extraction_mode: Literal["ocr", "hybrid", "metadata"] = "hybrid" force_url_result: bool = False force_file_extension: str | None = None return_ocr_data: bool = False @@ -285,6 +289,8 @@ def _retrieval_strategy() -> SearchStrategy[ReductoRetrieval]: def _settings_strategy() -> SearchStrategy[ReductoSettings]: + # force_url_result stays model-compatible but is not recorded until the + # response transform follows and downloads result.url. return_images: Final[SearchStrategy[list[ReductoReturnImage]]] = sampled_list_strategy(_REDUCTO_RETURN_IMAGE_GROUPS) page_ranges: Final = st.one_of( st.just(ReductoPageRange(start=1, end=1)), @@ -299,33 +305,50 @@ def _settings_strategy() -> SearchStrategy[ReductoSettings]: ), ) return st.one_of( + st.just(ReductoSettings(model="r-1")), st.sampled_from(("standard", "legacy")).map(lambda value: ReductoSettings(ocr_system=value)), - st.sampled_from(("hybrid", "ocr")).map(lambda value: ReductoSettings(extraction_mode=value)), - st.just(ReductoSettings(force_url_result=True)), + st.sampled_from(("hybrid", "ocr", "metadata")).map(lambda value: ReductoSettings(extraction_mode=value)), st.just(ReductoSettings(return_ocr_data=True)), return_images.map(lambda selected_images: ReductoSettings(return_images=selected_images)), + st.just(ReductoSettings(embed_pdf_metadata=True)), sampled_scalar_strategy((50, 100, 250)).map( lambda dpi: ReductoSettings(embed_pdf_metadata=True, embed_pdf_metadata_dpi=dpi) ), - sampled_scalar_strategy((300.0, 900.0)).map(lambda timeout: ReductoSettings(timeout=timeout)), + sampled_scalar_strategy((300.0,)).map(lambda timeout: ReductoSettings(timeout=timeout)), page_ranges.map(lambda page_range: ReductoSettings(page_range=page_range)), ) +def _reducto_v3_baseline( + route: _ReductoV3Route, + document: ReductoDocument, + inline_image_data_uri: str, +) -> ReductoParseV3SdkInput: + if route == "image": + inline_image: Final = ReductoImageUrlDocument.model_validate( + image_data_document(inline_image_data_uri).model_dump(mode="json") + ) + return ReductoParseV3SdkInput(model="reducto/parse-v3", document=inline_image) + if route == "unqualified": + return ReductoParseV3SdkInput( + model="parse-v3", + custom_llm_provider="reducto", + document=document, + ) + return ReductoParseV3SdkInput(model="reducto/parse-v3", document=document) + + def reducto_v3_input_strategy( - document: ReductoDocumentUrlDocument | None = None, + inline_image_data_uri: str, + document: ReductoDocument | None = None, ) -> SearchStrategy[ReductoParseV3SdkInput]: selected_document: Final = document or ReductoDocumentUrlDocument( type="document_url", document_url="reducto://fixture-document.pdf" ) + baseline_routes: Final[tuple[_ReductoV3Route, ...]] = ("qualified", "image", "unqualified") return st.one_of( - st.just(ReductoParseV3SdkInput(model="reducto/parse-v3", document=selected_document)), - st.just( - ReductoParseV3SdkInput( - model="parse-v3", - custom_llm_provider="reducto", - document=selected_document, - ) + st.sampled_from(baseline_routes).map( + lambda route: _reducto_v3_baseline(route, selected_document, inline_image_data_uri) ), _formatting_strategy().map( lambda formatting: ReductoParseV3SdkInput( @@ -351,33 +374,43 @@ def reducto_v3_input_strategy( ) +def _reducto_legacy_input( + route: _ReductoLegacyRoute, + document: ReductoDocument, +) -> ReductoParseLegacySdkInput: + if route == "unqualified": + return ReductoParseLegacySdkInput( + model="parse-legacy", + custom_llm_provider="reducto", + document=document, + ) + return ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=document) + + def reducto_legacy_input_strategy( - document: ReductoDocumentUrlDocument | None = None, + document: ReductoDocument | None = None, ) -> SearchStrategy[ReductoParseLegacySdkInput]: selected_document: Final = document or ReductoDocumentUrlDocument( type="document_url", document_url="reducto://fixture-document.pdf" ) - return st.sampled_from( - ( - ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=selected_document), - ReductoParseLegacySdkInput(model="parse-legacy", custom_llm_provider="reducto", document=selected_document), - ReductoParseLegacySdkInput(model="reducto/parse-legacy", document=selected_document, enhance={}), - ) - ) + routes: Final[tuple[_ReductoLegacyRoute, ...]] = ("qualified", "unqualified") + return st.sampled_from(routes).map(lambda route: _reducto_legacy_input(route, selected_document)) -def reducto_recording_targets(environ: Mapping[str, str], client: OcrFixtureClient) -> tuple[OcrRecordingTarget, ...]: +def reducto_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: api_key: Final = environ.get("REDUCTO_API_KEY") if not api_key: return () upstream_base: Final = environ.get("REDUCTO_API_BASE", _REDUCTO_API_BASE).rstrip("/") - document: Final = ReductoDocumentUrlDocument(type="document_url", document_url=fixture_pdf_data_uri()) + document: Final = ReductoDocumentUrlDocument(type="document_url", document_url=structured_pdf_data_uri()) invocation: Final = invoke_with_api_key(client, api_key) return ( OcrRecordingTarget( name="reducto-v3", provider_spec=ProviderSpec(upstream_base=upstream_base), - strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_v3_input_strategy(document)), + strategy=cast(SearchStrategy[OcrSdkInputBase], reducto_v3_input_strategy(inline_image_data_uri, document)), invocation=invocation, ), OcrRecordingTarget( diff --git a/tests/test_litellm/ocr/fixtures/vertex.py b/tests/test_litellm/ocr/fixtures/vertex.py index f2d894d6989..c7014bb75ec 100644 --- a/tests/test_litellm/ocr/fixtures/vertex.py +++ b/tests/test_litellm/ocr/fixtures/vertex.py @@ -12,15 +12,13 @@ from tests.test_litellm.ocr.fixtures.base import OcrDocument, OcrSdkInputBase from tests.test_litellm.ocr.fixtures.common import ( OcrFixtureClient, OcrRecordingTarget, + image_data_document, invoke_with_api_key, - public_document_strategy, sampled_scalar_strategy, ) from tests.test_litellm.ocr.fixtures.mistral import ( - MISTRAL_MODEL, MistralCompatibleOcrSdkInput, - MistralOcrSdkInput, - mistral_input_strategy, + mistral_input_values_strategy, ) VertexMistralModel = Literal["vertex_ai/mistral-ocr-2505"] @@ -48,34 +46,50 @@ class VertexDeepSeekOcrSdkInput(OcrSdkInputBase): def _as_vertex_mistral( - case_input: MistralOcrSdkInput, + values: dict[str, object], project: str, location: str, model: VertexMistralModel, ) -> VertexMistralOcrSdkInput: - values: Final = case_input.model_dump( - mode="python", - exclude={"boundary", "model", "custom_llm_provider"}, - exclude_unset=True, - ) return VertexMistralOcrSdkInput.model_validate( {**values, "model": model, "vertex_project": project, "vertex_location": location} ) +def vertex_mistral_input_strategy( + project: str, + location: str, + inline_image_data_uri: str, +) -> SearchStrategy[VertexMistralOcrSdkInput]: + return st.builds( + _as_vertex_mistral, + project=st.just(project), + location=st.just(location), + model=sampled_scalar_strategy(VERTEX_MISTRAL_MODELS), + values=mistral_input_values_strategy("2505", inline_image_data_uri), + ) + + @st.composite -def vertex_deepseek_input_strategy(draw: DrawFn, project: str, location: str) -> VertexDeepSeekOcrSdkInput: +def vertex_deepseek_input_strategy( + draw: DrawFn, project: str, location: str, inline_image_data_uri: str +) -> VertexDeepSeekOcrSdkInput: return VertexDeepSeekOcrSdkInput.model_validate( { "model": draw(sampled_scalar_strategy(VERTEX_DEEPSEEK_MODELS)), - "document": draw(public_document_strategy()), + # The current Vertex model card documents image input only. Keep + # the broader fixture model for existing recordings, but do not + # spend a paid request on the transform's unsupported PDF branch. + "document": image_data_document(inline_image_data_uri), "vertex_project": project, "vertex_location": location, } ) -def vertex_recording_targets(environ: Mapping[str, str], client: OcrFixtureClient) -> tuple[OcrRecordingTarget, ...]: +def vertex_recording_targets( + environ: Mapping[str, str], client: OcrFixtureClient, inline_image_data_uri: str +) -> tuple[OcrRecordingTarget, ...]: api_key: Final = environ.get("VERTEX_AI_API_KEY") project: Final = environ.get("VERTEXAI_PROJECT") or environ.get("VERTEX_PROJECT") location: Final = environ.get("VERTEXAI_LOCATION") or environ.get("VERTEX_LOCATION") or "us-central1" @@ -89,20 +103,17 @@ def vertex_recording_targets(environ: Mapping[str, str], client: OcrFixtureClien provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), strategy=cast( SearchStrategy[OcrSdkInputBase], - st.builds( - _as_vertex_mistral, - project=st.just(project), - location=st.just(location), - model=sampled_scalar_strategy(VERTEX_MISTRAL_MODELS), - case_input=mistral_input_strategy(MISTRAL_MODEL, feature_level="2505"), - ), + vertex_mistral_input_strategy(project, location, inline_image_data_uri), ), invocation=invocation, ), OcrRecordingTarget( name="vertex-deepseek", provider_spec=ProviderSpec(upstream_base=upstream_base.rstrip("/")), - strategy=cast(SearchStrategy[OcrSdkInputBase], vertex_deepseek_input_strategy(project, location)), + strategy=cast( + SearchStrategy[OcrSdkInputBase], + vertex_deepseek_input_strategy(project, location, inline_image_data_uri), + ), invocation=invocation, ), ) diff --git a/tests/test_litellm/ocr/test_fixture_models.py b/tests/test_litellm/ocr/test_fixture_models.py index 8f95c6a0df2..34856a7bee1 100644 --- a/tests/test_litellm/ocr/test_fixture_models.py +++ b/tests/test_litellm/ocr/test_fixture_models.py @@ -1,11 +1,16 @@ from __future__ import annotations +import base64 from collections.abc import Callable from datetime import date from pathlib import Path from typing import Final, TypeVar, cast +from unittest.mock import patch +from urllib.parse import parse_qs, urlparse +import httpx import pytest +import respx from hypothesis import find, given, settings from hypothesis import strategies as st from hypothesis.strategies import DataObject, SearchStrategy @@ -13,18 +18,25 @@ from pydantic import BaseModel, ConfigDict, JsonValue, TypeAdapter, ValidationEr from litellm.llms.azure_ai.ocr.document_intelligence.transformation import AzureDocumentIntelligenceOCRConfig from litellm.llms.azure_ai.ocr.transformation import AzureAIOCRConfig -from litellm.llms.base_llm.ocr.transformation import BaseOCRConfig +from litellm.llms.base_llm.ocr.transformation import ( + BaseOCRConfig, + DocumentType, + OCRRequestData, +) from litellm.llms.mistral.ocr.transformation import MistralOCRConfig from litellm.llms.reducto.ocr.transformation import ReductoParseLegacyConfig, ReductoParseV3Config from litellm.llms.vertex_ai.ocr.deepseek_transformation import VertexAIDeepSeekOCRConfig from litellm.llms.vertex_ai.ocr.transformation import VertexAIOCRConfig +from tests.route_parity.fixtures.media import structured_pdf_data_uri from tests.test_litellm.ocr.conftest import ocr_fixture_marks from tests.test_litellm.ocr.fixtures.azure import ( AZURE_DOCUMENT_INTELLIGENCE_MODELS, + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, AZURE_MISTRAL_MODELS, AzureDocumentIntelligenceOcrSdkInput, AzureMistralOcrSdkInput, azure_document_intelligence_input_strategy, + azure_mistral_input_strategy, ) from tests.test_litellm.ocr.fixtures.base import ( DocumentUrlDocument, @@ -42,6 +54,7 @@ from tests.test_litellm.ocr.fixtures.reducto import ( ReductoChunking, ReductoDocumentUrlDocument, ReductoFormatting, + ReductoImageUrlDocument, ReductoPageRange, ReductoParseLegacySdkInput, ReductoParseV3SdkInput, @@ -56,6 +69,7 @@ from tests.test_litellm.ocr.fixtures.vertex import ( VertexDeepSeekOcrSdkInput, VertexMistralOcrSdkInput, vertex_deepseek_input_strategy, + vertex_mistral_input_strategy, ) COMMON_FIELDS: Final = frozenset( @@ -109,15 +123,79 @@ _MISTRAL_OPTION_GROUPS: Final = frozenset( "table_format", "confidence_scores_granularity", "include_blocks", - "id", ) ), frozenset({"document_annotation_format", "document_annotation_prompt"}), frozenset({"include_blocks", "confidence_scores_granularity"}), } ) +_MISTRAL_2505_OPTION_GROUPS: Final = frozenset( + { + frozenset[str](), + *( + frozenset({field}) + for field in ( + "pages", + "include_image_base64", + "image_limit", + "image_min_size", + "bbox_annotation_format", + "document_annotation_format", + "confidence_scores_granularity", + ) + ), + frozenset({"document_annotation_format", "document_annotation_prompt"}), + } +) +_AZURE_MISTRAL_OPTION_GROUPS: Final = _MISTRAL_2505_OPTION_GROUPS - { + frozenset({"document_annotation_format", "document_annotation_prompt"}) +} +_REDUCTO_FORMATTING_INCLUDE_GROUPS: Final = ( + (), + ("hyperlinks",), + ("change_tracking", "highlight", "comments"), + ("signatures", "ignore_watermarks"), +) +_REDUCTO_FILTER_BLOCK_GROUPS: Final = ( + (), + ("Header",), + ("Header", "Footer", "Page Number"), + ("Figure", "Table", "Key Value"), +) +_REDUCTO_RETURN_IMAGE_GROUPS: Final = ( + (), + ("figure",), + ("table",), + ("page",), + ("figure", "table"), +) _FIND_SETTINGS: Final = settings(max_examples=2_000, deadline=None, derandomize=True, database=None) _FixtureInputT = TypeVar("_FixtureInputT") +INLINE_IMAGE_DATA_URI: Final = "data:image/png;base64,dGVzdA==" +_MapOcrParams = Callable[[dict[str, object], dict[str, object], str], dict[str, object]] +_TransformOcrRequest = Callable[ + [str, DocumentType, dict[str, object], dict[str, object]], + OCRRequestData, +] +_GetCompleteUrl = Callable[[str | None, str, dict[str, object]], str] + + +def _transform_with_stubbed_download( + transform_request: _TransformOcrRequest, + model: str, + document: DocumentType, + mapped: dict[str, object], +) -> OCRRequestData: + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + if source.startswith("data:"): + return transform_request(model, document, mapped, {}) + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + with respx.mock(assert_all_called=False) as router: + router.route(method="GET").mock( + return_value=httpx.Response(200, content=b"\x00", headers={"content-type": media_type}) + ) + return transform_request(model, document, mapped, {}) def _find_fixture( @@ -127,6 +205,49 @@ def _find_fixture( return find(strategy, predicate, settings=_FIND_SETTINGS) +def _document_transport(document: ImageUrlDocument | DocumentUrlDocument) -> tuple[str, str]: + if isinstance(document, ImageUrlDocument): + source: Final = document.image_url.url if isinstance(document.image_url, ImageUrlValue) else document.image_url + return document.type, "data" if source.startswith("data:") else "remote" + return document.type, "data" if document.document_url.startswith("data:") else "remote" + + +def _normalized_azure_pages(pages: object) -> str: + if isinstance(pages, str): + return pages.replace(" ", "") + assert isinstance(pages, list) + raw_pages: Final = cast(list[object], pages) + if all(isinstance(page, int) for page in raw_pages): + integer_pages: Final = cast(list[int], raw_pages) + return ",".join(str(page + 1) for page in sorted(set(integer_pages))) + string_pages: Final = cast(list[str], raw_pages) + return ",".join(page.strip() for page in string_pages) + + +def test_structured_pdf_exercises_semantic_ocr_features() -> None: + encoded: Final = structured_pdf_data_uri().partition(",")[2] + pdf: Final = base64.b64decode(encoded, validate=True) + + assert pdf.startswith(b"%PDF-1.") + assert b"/Count 5" in pdf + assert pdf.count(b"/Subtype /Image") == 3 + assert all( + marker in pdf + for marker in ( + b"/Width 120", + b"/Width 320", + b"/Width 360", + b"/Subtype /Highlight", + b"/Subtype /Link", + b"/Subtype /Text", + b"/Title (Quarterly Operations Report)", + ) + ) + assert b"Invoice Number: INV-2048" in pdf + assert b"Formula: gross margin" in pdf + assert b"Approved by: Jordan Lee" in pdf + + class _ModelRegistryEntry(BaseModel): model_config = ConfigDict(frozen=True, extra="ignore") @@ -349,6 +470,30 @@ def test_vertex_deepseek_request_uses_single_provider_namespace(model: str) -> N assert data["model"] == "deepseek-ai/deepseek-ocr-maas" +@pytest.mark.parametrize( + "document", + ( + {"type": "image_url", "image_url": "data:image/png;base64,AA=="}, + {"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, + ), +) +def test_vertex_deepseek_request_maps_both_document_types_to_image_content( + document: DocumentType, +) -> None: + request: Final = VertexAIDeepSeekOCRConfig().transform_ocr_request( # pyright: ignore[reportUnknownMemberType] + model="deepseek-ai/deepseek-ocr-maas", + document=document, + optional_params={}, + headers={}, + ) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + data: Final = cast(dict[str, object], request.data) + messages: Final = cast(list[dict[str, object]], data["messages"]) + content: Final = cast(list[dict[str, object]], messages[0]["content"]) + assert content == [{"type": "image_url", "image_url": document[source_key]}] + + @pytest.mark.parametrize( "sdk_input", ( @@ -434,12 +579,12 @@ def test_reducto_nested_constraints() -> None: @settings(max_examples=100, deadline=None) @given(model=st.sampled_from(MISTRAL_MODELS), data=st.data()) def test_mistral_strategy_only_generates_bounded_valid_sdk_inputs(model: str, data: DataObject) -> None: - sdk_input: Final = data.draw(mistral_input_strategy(model)) + sdk_input: Final = data.draw(mistral_input_strategy(model, INLINE_IMAGE_DATA_URI)) assert MistralOcrSdkInput.model_validate(sdk_input.canonical_input()) == sdk_input optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} assert optional_fields in _MISTRAL_OPTION_GROUPS if sdk_input.pages is not None: - assert sdk_input.pages in ([0], [0, 1]) + assert sdk_input.pages in ([0], [0, 1], "0-2") if sdk_input.image_limit is not None: assert sdk_input.image_limit == 1 if sdk_input.image_min_size is not None: @@ -454,6 +599,46 @@ def test_mistral_strategy_only_generates_bounded_valid_sdk_inputs(model: str, da assert optional_fields.isdisjoint({"extract_header", "extract_footer", "table_format"}) if model not in _MISTRAL_4_OR_NEWER: assert "include_blocks" not in optional_fields + assert not isinstance(sdk_input.pages, str) + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + "transport", + ( + ("image_url", "remote"), + ("image_url", "data"), + ("document_url", "remote"), + ("document_url", "data"), + ), +) +def test_mistral_strategy_reaches_every_document_transform_branch(transport: tuple[str, str]) -> None: + sdk_input: Final = _find_fixture( + mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI), + lambda candidate: _document_transport(candidate.document) == transport, + ) + + assert _document_transport(sdk_input.document) == transport + + +@settings(max_examples=100, deadline=None) +@given(sdk_input=mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI)) +def test_mistral_strategy_values_survive_the_request_transform(sdk_input: MistralOcrSdkInput) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = MistralOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + request: Final = transform_request(model, document, mapped, {}) + request_data: Final = cast(dict[str, object], request.data) + + assert mapped == optional_params + assert request_data == {"model": model, "document": document, **optional_params} @pytest.mark.parametrize( @@ -461,6 +646,7 @@ def test_mistral_strategy_only_generates_bounded_valid_sdk_inputs(model: str, da ( ("pages", [0]), ("pages", [0, 1]), + ("pages", "0-2"), ("include_image_base64", False), ("include_image_base64", True), ("image_limit", 1), @@ -476,12 +662,11 @@ def test_mistral_strategy_only_generates_bounded_valid_sdk_inputs(model: str, da ("confidence_scores_granularity", "block"), ("include_blocks", False), ("include_blocks", True), - ("id", "case-1"), ), ) def test_mistral_strategy_reaches_every_finite_scalar_value(field: str, value: object) -> None: sdk_input: Final = _find_fixture( - mistral_input_strategy("mistral/mistral-ocr-4-1"), + mistral_input_strategy("mistral/mistral-ocr-4-1", INLINE_IMAGE_DATA_URI), lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, ) @@ -489,20 +674,22 @@ def test_mistral_strategy_reaches_every_finite_scalar_value(field: str, value: o @settings(max_examples=50, deadline=None) -@given(sdk_input=reducto_v3_input_strategy()) +@given(sdk_input=reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI)) def test_reducto_v3_strategy_only_generates_bounded_valid_sdk_inputs(sdk_input: ReductoParseV3SdkInput) -> None: assert ReductoParseV3SdkInput.model_validate(sdk_input.canonical_input()) == sdk_input option_groups: Final = frozenset(sdk_input.model_fields_set) & {"formatting", "retrieval", "settings"} assert len(option_groups) <= 1 if "formatting" in option_groups: - assert len(sdk_input.formatting.model_fields_set) == 1 - assert sdk_input.formatting.table_output_format in {"dynamic", "html", "md", "json", "csv", "jsonbbox"} - assert tuple(sdk_input.formatting.include) in { - (), - ("hyperlinks",), - ("change_tracking", "highlight", "comments"), - ("signatures", "ignore_watermarks"), - } + formatting_fields: Final = frozenset(sdk_input.formatting.model_fields_set) + assert len(formatting_fields) == 1 + if "table_output_format" in formatting_fields: + assert sdk_input.formatting.table_output_format in {"dynamic", "html", "md", "json", "csv", "jsonbbox"} + if "add_page_markers" in formatting_fields: + assert sdk_input.formatting.add_page_markers in {False, True} + if "merge_tables" in formatting_fields: + assert sdk_input.formatting.merge_tables in {False, True} + if "include" in formatting_fields: + assert tuple(sdk_input.formatting.include) in _REDUCTO_FORMATTING_INCLUDE_GROUPS if "retrieval" in option_groups: retrieval_fields: Final = frozenset(sdk_input.retrieval.model_fields_set) assert retrieval_fields in { @@ -511,39 +698,123 @@ def test_reducto_v3_strategy_only_generates_bounded_valid_sdk_inputs(sdk_input: frozenset({"chunking", "embedding_optimized"}), } chunking: Final = sdk_input.retrieval.chunking - if chunking.chunk_size is not None or chunking.chunk_overlap != 0: + if "chunking" in retrieval_fields: + assert chunking.chunk_mode in {"variable", "section", "page", "disabled", "block", "page_sections"} + assert chunking.chunk_size in {None, 250, 1000, 1500} + assert chunking.chunk_overlap in {0, 32, 128} + if chunking.chunk_size is not None or chunking.chunk_overlap: assert chunking.chunk_mode == "variable" + if chunking.chunk_overlap: + assert chunking.chunk_size == 1000 + if "filter_blocks" in retrieval_fields: + assert tuple(sdk_input.retrieval.filter_blocks) in _REDUCTO_FILTER_BLOCK_GROUPS if "embedding_optimized" in retrieval_fields: assert chunking.chunk_mode == "variable" + assert chunking.chunk_size is None + assert chunking.chunk_overlap == 0 + assert sdk_input.retrieval.embedding_optimized in {False, True} if "settings" in option_groups: settings_fields: Final = frozenset(sdk_input.settings.model_fields_set) assert settings_fields in { + frozenset({"model"}), frozenset({"ocr_system"}), frozenset({"extraction_mode"}), - frozenset({"force_url_result"}), frozenset({"return_ocr_data"}), frozenset({"return_images"}), + frozenset({"embed_pdf_metadata"}), frozenset({"embed_pdf_metadata", "embed_pdf_metadata_dpi"}), frozenset({"timeout"}), frozenset({"page_range"}), } - assert "persist_results" not in settings_fields + assert settings_fields.isdisjoint( + { + "force_url_result", + "force_file_extension", + "persist_results", + "tenant_throttling", + "document_password", + "hybrid_vpc", + } + ) + if "model" in settings_fields: + assert sdk_input.settings.model == "r-1" + if "ocr_system" in settings_fields: + assert sdk_input.settings.ocr_system in {"standard", "legacy"} + if "extraction_mode" in settings_fields: + assert sdk_input.settings.extraction_mode in {"hybrid", "ocr", "metadata"} + if "return_ocr_data" in settings_fields: + assert sdk_input.settings.return_ocr_data is True + if "return_images" in settings_fields: + assert tuple(sdk_input.settings.return_images) in _REDUCTO_RETURN_IMAGE_GROUPS if "embed_pdf_metadata_dpi" in settings_fields: assert sdk_input.settings.embed_pdf_metadata is True assert sdk_input.settings.embed_pdf_metadata_dpi in {50, 100, 250} + if "timeout" in settings_fields: + assert sdk_input.settings.timeout == 300.0 if sdk_input.settings.page_range is not None: - ranges: Final = ( - sdk_input.settings.page_range - if isinstance(sdk_input.settings.page_range, list) - else [sdk_input.settings.page_range] + dumped_range: Final = cast( + dict[str, object], sdk_input.settings.model_dump(mode="json", exclude_unset=True) + )["page_range"] + assert dumped_range in ( + {"start": 1, "end": 1}, + {"start": 1, "end": 3}, + [{"start": 1, "end": 2}, {"start": 4, "end": 5}], ) - assert all(isinstance(page_range, ReductoPageRange) for page_range in ranges) + + +@settings(max_examples=60, deadline=None) +@given(sdk_input=reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_reducto_v3_strategy_values_survive_the_request_transform(sdk_input: ReductoParseV3SdkInput) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = { + name: value for name, value in sdk_kwargs.items() if name not in {"model", "document", "custom_llm_provider"} + } + config: Final = ReductoParseV3Config() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + with patch.object(config, "_ensure_file_id_sync", return_value="reducto://fixture-document.pdf"): + request: Final = transform_request(model, document, mapped, {}) + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "input": "reducto://fixture-document.pdf", + **optional_params, + } + + +def test_reducto_v3_strategy_reaches_image_upload_branch_without_options() -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: isinstance(candidate.document, ReductoImageUrlDocument), + ) + + assert isinstance(sdk_input.document, ReductoImageUrlDocument) + assert sdk_input.document.image_url.startswith("data:image/") + assert sdk_input.model_fields_set == {"model", "document"} + + +@pytest.mark.parametrize( + ("model", "provider"), + (("reducto/parse-v3", None), ("parse-v3", "reducto")), +) +def test_reducto_v3_strategy_reaches_every_routing_form(model: str, provider: str | None) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: candidate.model == model and candidate.custom_llm_provider == provider, + ) + + assert sdk_input.model == model + assert sdk_input.custom_llm_provider == provider @pytest.mark.parametrize("table_format", ("dynamic", "html", "md", "json", "csv", "jsonbbox")) def test_reducto_v3_strategy_reaches_every_table_format(table_format: str) -> None: sdk_input: Final = _find_fixture( - reducto_v3_input_strategy(), + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), lambda candidate: ( "formatting" in candidate.model_fields_set and "table_output_format" in candidate.formatting.model_fields_set @@ -554,10 +825,46 @@ def test_reducto_v3_strategy_reaches_every_table_format(table_format: str) -> No assert sdk_input.formatting.table_output_format == table_format +@pytest.mark.parametrize( + ("field", "value"), + ( + ("add_page_markers", False), + ("add_page_markers", True), + ("merge_tables", False), + ("merge_tables", True), + ), +) +def test_reducto_v3_strategy_reaches_every_formatting_boolean(field: str, value: bool) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "formatting" in candidate.model_fields_set + and field in candidate.formatting.model_fields_set + and getattr(candidate.formatting, field) is value + ), + ) + + assert getattr(sdk_input.formatting, field) is value + + +@pytest.mark.parametrize("include", _REDUCTO_FORMATTING_INCLUDE_GROUPS) +def test_reducto_v3_strategy_reaches_every_formatting_include(include: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "formatting" in candidate.model_fields_set + and "include" in candidate.formatting.model_fields_set + and tuple(candidate.formatting.include) == include + ), + ) + + assert tuple(sdk_input.formatting.include) == include + + @pytest.mark.parametrize("chunk_mode", ("variable", "section", "page", "disabled", "block", "page_sections")) def test_reducto_v3_strategy_reaches_every_chunk_mode(chunk_mode: str) -> None: sdk_input: Final = _find_fixture( - reducto_v3_input_strategy(), + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), lambda candidate: ( "retrieval" in candidate.model_fields_set and "chunking" in candidate.retrieval.model_fields_set @@ -571,7 +878,7 @@ def test_reducto_v3_strategy_reaches_every_chunk_mode(chunk_mode: str) -> None: @pytest.mark.parametrize("chunk_size", (250, 1000, 1500)) def test_reducto_v3_strategy_reaches_every_chunk_size(chunk_size: int) -> None: sdk_input: Final = _find_fixture( - reducto_v3_input_strategy(), + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), lambda candidate: candidate.retrieval.chunking.chunk_size == chunk_size, ) @@ -579,10 +886,51 @@ def test_reducto_v3_strategy_reaches_every_chunk_size(chunk_size: int) -> None: assert sdk_input.retrieval.chunking.chunk_size == chunk_size +@pytest.mark.parametrize("chunk_overlap", (32, 128)) +def test_reducto_v3_strategy_reaches_every_chunk_overlap(chunk_overlap: int) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: candidate.retrieval.chunking.chunk_overlap == chunk_overlap, + ) + + assert sdk_input.retrieval.chunking.chunk_mode == "variable" + assert sdk_input.retrieval.chunking.chunk_size == 1000 + assert sdk_input.retrieval.chunking.chunk_overlap == chunk_overlap + + +@pytest.mark.parametrize("filter_blocks", _REDUCTO_FILTER_BLOCK_GROUPS) +def test_reducto_v3_strategy_reaches_every_filter_block_group(filter_blocks: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "retrieval" in candidate.model_fields_set + and "filter_blocks" in candidate.retrieval.model_fields_set + and tuple(candidate.retrieval.filter_blocks) == filter_blocks + ), + ) + + assert tuple(sdk_input.retrieval.filter_blocks) == filter_blocks + + +@pytest.mark.parametrize("embedding_optimized", (False, True)) +def test_reducto_v3_strategy_reaches_every_embedding_setting(embedding_optimized: bool) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "retrieval" in candidate.model_fields_set + and "embedding_optimized" in candidate.retrieval.model_fields_set + and candidate.retrieval.embedding_optimized is embedding_optimized + ), + ) + + assert sdk_input.retrieval.chunking.chunk_mode == "variable" + assert sdk_input.retrieval.embedding_optimized is embedding_optimized + + @pytest.mark.parametrize("dpi", (50, 100, 250)) def test_reducto_v3_strategy_reaches_every_metadata_dpi(dpi: int) -> None: sdk_input: Final = _find_fixture( - reducto_v3_input_strategy(), + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), lambda candidate: ( "settings" in candidate.model_fields_set and "embed_pdf_metadata_dpi" in candidate.settings.model_fields_set @@ -594,6 +942,58 @@ def test_reducto_v3_strategy_reaches_every_metadata_dpi(dpi: int) -> None: assert sdk_input.settings.embed_pdf_metadata_dpi == dpi +def test_reducto_v3_strategy_reaches_metadata_with_default_dpi_omitted() -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set and candidate.settings.model_fields_set == {"embed_pdf_metadata"} + ), + ) + + assert sdk_input.settings.embed_pdf_metadata is True + assert "embed_pdf_metadata_dpi" not in sdk_input.settings.model_fields_set + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("model", "r-1"), + ("ocr_system", "standard"), + ("ocr_system", "legacy"), + ("extraction_mode", "hybrid"), + ("extraction_mode", "ocr"), + ("extraction_mode", "metadata"), + ("return_ocr_data", True), + ("timeout", 300.0), + ), +) +def test_reducto_v3_strategy_reaches_every_scalar_setting(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set + and field in candidate.settings.model_fields_set + and getattr(candidate.settings, field) == value + ), + ) + + assert getattr(sdk_input.settings, field) == value + + +@pytest.mark.parametrize("return_images", _REDUCTO_RETURN_IMAGE_GROUPS) +def test_reducto_v3_strategy_reaches_every_return_image_group(return_images: tuple[str, ...]) -> None: + sdk_input: Final = _find_fixture( + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: ( + "settings" in candidate.model_fields_set + and "return_images" in candidate.settings.model_fields_set + and tuple(candidate.settings.return_images) == return_images + ), + ) + + assert tuple(sdk_input.settings.return_images) == return_images + + @pytest.mark.parametrize( "page_range", ( @@ -604,7 +1004,7 @@ def test_reducto_v3_strategy_reaches_every_metadata_dpi(dpi: int) -> None: ) def test_reducto_v3_strategy_reaches_every_page_range_shape(page_range: object) -> None: sdk_input: Final = _find_fixture( - reducto_v3_input_strategy(), + reducto_v3_input_strategy(INLINE_IMAGE_DATA_URI), lambda candidate: ( cast( dict[str, object], @@ -621,6 +1021,128 @@ def test_reducto_v3_strategy_reaches_every_page_range_shape(page_range: object) @given(sdk_input=reducto_legacy_input_strategy()) def test_reducto_legacy_strategy_generates_valid_litellm_inputs(sdk_input: ReductoParseLegacySdkInput) -> None: assert ReductoParseLegacySdkInput.model_validate(sdk_input.canonical_input()) == sdk_input + assert "enhance" not in sdk_input.model_fields_set + + +@settings(max_examples=10, deadline=None) +@given(sdk_input=reducto_legacy_input_strategy()) +def test_reducto_legacy_strategy_values_survive_the_request_transform( + sdk_input: ReductoParseLegacySdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + config: Final = ReductoParseLegacyConfig() + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + + with patch.object(config, "_ensure_file_id_sync", return_value="reducto://fixture-document.pdf"): + request: Final = transform_request(model, document, {}, {}) + + assert cast(dict[str, object], request.data) == { + "document_url": "reducto://fixture-document.pdf", + } + + +@pytest.mark.parametrize( + ("model", "provider"), + (("reducto/parse-legacy", None), ("parse-legacy", "reducto")), +) +def test_reducto_legacy_strategy_reaches_every_routing_form(model: str, provider: str | None) -> None: + sdk_input: Final = _find_fixture( + reducto_legacy_input_strategy(), + lambda candidate: candidate.model == model and candidate.custom_llm_provider == provider, + ) + + assert sdk_input.model == model + assert sdk_input.custom_llm_provider == provider + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_azure_mistral_strategy_is_contained_to_gateway_capabilities( + sdk_input: AzureMistralOcrSdkInput, +) -> None: + optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} + + assert optional_fields in _AZURE_MISTRAL_OPTION_GROUPS + assert optional_fields.isdisjoint( + { + "document_annotation_prompt", + "extract_header", + "extract_footer", + "table_format", + "include_blocks", + "id", + } + ) + assert not isinstance(sdk_input.pages, str) + assert sdk_input.confidence_scores_granularity in {None, "page", "word"} + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [0, 1]), + ("include_image_base64", False), + ("include_image_base64", True), + ("image_limit", 1), + ("image_min_size", 300), + ("confidence_scores_granularity", "page"), + ("confidence_scores_granularity", "word"), + ), +) +def test_azure_mistral_strategy_reaches_every_gateway_scalar(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +@pytest.mark.parametrize("field", ("bbox_annotation_format", "document_annotation_format")) +def test_azure_mistral_strategy_reaches_every_gateway_schema(field: str) -> None: + sdk_input: Final = _find_fixture( + azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI), + lambda candidate: frozenset(candidate.model_fields_set) - {"model", "document"} == frozenset({field}), + ) + + assert frozenset(sdk_input.model_fields_set) - {"model", "document"} == {field} + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_mistral_input_strategy(INLINE_IMAGE_DATA_URI)) +def test_azure_mistral_strategy_exercises_url_conversion_and_inline_bypass( + sdk_input: AzureMistralOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = AzureAIOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + request: Final = _transform_with_stubbed_download(transform_request, model, document, mapped) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + expected_document: Final = dict(document) + if not source.startswith("data:"): + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + expected_document[source_key] = f"data:{media_type};base64,AA==" + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "model": model, + "document": expected_document, + **optional_params, + } @settings(max_examples=30, deadline=None) @@ -629,16 +1151,18 @@ def test_azure_document_intelligence_strategy_only_generates_litellm_inputs( sdk_input: AzureDocumentIntelligenceOcrSdkInput, ) -> None: assert sdk_input.req_format == "litellm" + assert sdk_input.model in AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS assert "boundary" not in sdk_input.as_sdk_kwargs() optional_fields: Final = frozenset(sdk_input.model_fields_set) - {"model", "document"} assert optional_fields in { frozenset[str](), frozenset({"pages"}), frozenset({"features"}), + frozenset({"pages", "features"}), frozenset({"req_format"}), } if sdk_input.pages is not None: - assert sdk_input.pages in ([0], [0, 1], "1", "1,2", "1-2") + assert sdk_input.pages in ([0], [2, 0, 0, 1], ["1", "2-4"], "1-4, 5", [0, 1]) if isinstance(sdk_input.features, list): assert tuple(sdk_input.features) in { ("languages",), @@ -647,27 +1171,70 @@ def test_azure_document_intelligence_strategy_only_generates_litellm_inputs( ("formulas",), ("styleFont",), ("keyValuePairs",), + ("languages", "styleFont"), } if isinstance(sdk_input.features, str): - assert sdk_input.features == "languages,styleFont" + assert sdk_input.features == "languages, styleFont" + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=azure_document_intelligence_input_strategy()) +def test_azure_document_intelligence_strategy_exercises_request_transform( + sdk_input: AzureDocumentIntelligenceOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = {name: value for name, value in sdk_kwargs.items() if name not in {"model", "document"}} + config: Final = AzureDocumentIntelligenceOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + get_complete_url: Final = cast(_GetCompleteUrl, config.get_complete_url) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + url: Final = get_complete_url("https://document.example", model, mapped) + query: Final = parse_qs(urlparse(url).query) + request: Final = transform_request(model, document, mapped, {}) + + if sdk_input.pages is None: + assert "pages" not in mapped + assert "pages" not in query + else: + expected_pages: Final = _normalized_azure_pages(sdk_input.pages) + assert mapped["pages"] == expected_pages + assert query["pages"] == [expected_pages] + if sdk_input.features is None: + assert "features" not in mapped + assert "features" not in query + else: + raw_features: Final = ( + sdk_input.features.split(",") if isinstance(sdk_input.features, str) else sdk_input.features + ) + expected_features: Final = ",".join(feature.strip() for feature in raw_features) + assert mapped["features"] == expected_features + assert query["features"] == [expected_features] + + source: Final = document["document_url"] if document["type"] == "document_url" else document["image_url"] + assert isinstance(source, str) + expected_body: Final = ( + {"base64Source": source.partition(",")[2]} if source.startswith("data:") else {"urlSource": source} + ) + assert cast(dict[str, object], request.data) == expected_body @pytest.mark.parametrize( ("field", "value"), ( ("pages", [0]), - ("pages", [0, 1]), - ("pages", "1"), - ("pages", "1,2"), - ("pages", "1-2"), + ("pages", [2, 0, 0, 1]), + ("pages", ["1", "2-4"]), + ("pages", "1-4, 5"), ("features", ["languages"]), ("features", ["ocrHighResolution"]), ("features", ["barcodes"]), ("features", ["formulas"]), ("features", ["styleFont"]), ("features", ["keyValuePairs"]), - ("features", "languages,styleFont"), - ("req_format", "litellm"), + ("features", "languages, styleFont"), ), ) def test_azure_document_intelligence_strategy_reaches_every_finite_value(field: str, value: object) -> None: @@ -679,10 +1246,142 @@ def test_azure_document_intelligence_strategy_reaches_every_finite_value(field: assert getattr(sdk_input, field) == value +def test_azure_document_intelligence_strategy_reaches_combined_query_branch() -> None: + sdk_input: Final = _find_fixture( + azure_document_intelligence_input_strategy(), + lambda candidate: {"pages", "features"}.issubset(candidate.model_fields_set), + ) + + assert sdk_input.pages == [0, 1] + assert sdk_input.features == ["languages", "styleFont"] + + +@pytest.mark.parametrize( + "transport", + (("document_url", "data"), ("image_url", "remote")), +) +def test_azure_document_intelligence_strategy_reaches_body_source_branches( + transport: tuple[str, str], +) -> None: + sdk_input: Final = _find_fixture( + azure_document_intelligence_input_strategy(), + lambda candidate: _document_transport(candidate.document) == transport, + ) + + assert _document_transport(sdk_input.document) == transport + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) +def test_vertex_mistral_strategy_is_contained_to_2505_capabilities( + sdk_input: VertexMistralOcrSdkInput, +) -> None: + optional_fields: Final = frozenset(sdk_input.model_fields_set) - { + "model", + "document", + "vertex_project", + "vertex_location", + } + + assert optional_fields in _MISTRAL_2505_OPTION_GROUPS + assert optional_fields.isdisjoint({"extract_header", "extract_footer", "table_format", "include_blocks", "id"}) + assert not isinstance(sdk_input.pages, str) + assert sdk_input.confidence_scores_granularity in {None, "page", "word"} + if optional_fields: + assert isinstance(sdk_input.document, DocumentUrlDocument) + assert sdk_input.document.document_url == structured_pdf_data_uri() + + +@pytest.mark.parametrize( + ("field", "value"), + ( + ("pages", [0]), + ("pages", [0, 1]), + ("include_image_base64", False), + ("include_image_base64", True), + ("image_limit", 1), + ("image_min_size", 300), + ("confidence_scores_granularity", "page"), + ("confidence_scores_granularity", "word"), + ), +) +def test_vertex_mistral_strategy_reaches_every_2505_scalar(field: str, value: object) -> None: + sdk_input: Final = _find_fixture( + vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: field in candidate.model_fields_set and getattr(candidate, field) == value, + ) + + assert getattr(sdk_input, field) == value + + +@pytest.mark.parametrize( + "fields", + ( + frozenset({"bbox_annotation_format"}), + frozenset({"document_annotation_format"}), + frozenset({"document_annotation_format", "document_annotation_prompt"}), + ), +) +def test_vertex_mistral_strategy_reaches_every_2505_schema_group(fields: frozenset[str]) -> None: + sdk_input: Final = _find_fixture( + vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: ( + frozenset(candidate.model_fields_set) - {"model", "document", "vertex_project", "vertex_location"} == fields + ), + ) + + assert frozenset(sdk_input.model_fields_set) - {"model", "document", "vertex_project", "vertex_location"} == fields + + +@settings(max_examples=50, deadline=None) +@given(sdk_input=vertex_mistral_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) +def test_vertex_mistral_strategy_exercises_url_conversion_and_inline_bypass( + sdk_input: VertexMistralOcrSdkInput, +) -> None: + sdk_kwargs: Final = sdk_input.as_sdk_kwargs() + model: Final = cast(str, sdk_kwargs["model"]) + document: Final = cast(DocumentType, sdk_kwargs["document"]) + optional_params: Final = { + name: value + for name, value in sdk_kwargs.items() + if name not in {"model", "document", "vertex_project", "vertex_location"} + } + config: Final = VertexAIOCRConfig() + map_params: Final = cast(_MapOcrParams, config.map_ocr_params) + transform_request: Final = cast(_TransformOcrRequest, config.transform_ocr_request) + mapped: Final = map_params(optional_params, {}, model) + + request: Final = _transform_with_stubbed_download(transform_request, model, document, mapped) + + source_key: Final = "image_url" if document["type"] == "image_url" else "document_url" + source: Final = document[source_key] + expected_document: Final = dict(document) + if not source.startswith("data:"): + media_type: Final = "image/png" if document["type"] == "image_url" else "application/pdf" + expected_document[source_key] = f"data:{media_type};base64,AA==" + + assert mapped == optional_params + assert cast(dict[str, object], request.data) == { + "model": model, + "document": expected_document, + **optional_params, + } + + @settings(max_examples=30, deadline=None) -@given(sdk_input=vertex_deepseek_input_strategy("project-1", "us-central1")) +@given(sdk_input=vertex_deepseek_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI)) def test_vertex_deepseek_strategy_only_generates_litellm_inputs( sdk_input: VertexDeepSeekOcrSdkInput, ) -> None: assert sdk_input.vertex_project == "project-1" assert "boundary" not in sdk_input.as_sdk_kwargs() + assert _document_transport(sdk_input.document) == ("image_url", "data") + + +def test_vertex_deepseek_strategy_reaches_documented_image_branch() -> None: + sdk_input: Final = _find_fixture( + vertex_deepseek_input_strategy("project-1", "us-central1", INLINE_IMAGE_DATA_URI), + lambda candidate: _document_transport(candidate.document) == ("image_url", "data"), + ) + + assert _document_transport(sdk_input.document) == ("image_url", "data") diff --git a/tests/test_litellm/ocr/test_record_fixtures.py b/tests/test_litellm/ocr/test_record_fixtures.py index edf1592655f..fafbef5c760 100644 --- a/tests/test_litellm/ocr/test_record_fixtures.py +++ b/tests/test_litellm/ocr/test_record_fixtures.py @@ -1,25 +1,29 @@ from __future__ import annotations import queue -from collections.abc import Callable +from collections.abc import Callable, Mapping from dataclasses import dataclass from pathlib import Path -from typing import Final +from typing import Final, cast import pytest from hypothesis import find, settings from hypothesis.strategies import SearchStrategy from tests.route_parity.fixtures.inputs import generate_case_inputs +from tests.route_parity.fixtures.media import structured_pdf_data_uri from tests.route_parity.fixtures.pipeline import parse_recording_args from tests.test_litellm.ocr.fixtures.azure import ( - AZURE_DOCUMENT_INTELLIGENCE_MODELS, + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, AZURE_MISTRAL_MODELS, ) from tests.test_litellm.ocr.fixtures.base import OcrSdkInputBase +from tests.test_litellm.ocr.fixtures.common import OcrFixtureClient, OcrRecordingTarget from tests.test_litellm.ocr.fixtures.mistral import MISTRAL_MODELS from tests.test_litellm.ocr.fixtures.record import ( - discover_targets, + discover_targets as discover_targets_with_media, +) +from tests.test_litellm.ocr.fixtures.record import ( require_targets, ) from tests.test_litellm.ocr.fixtures.reducto import REDUCTO_LEGACY_MODELS, REDUCTO_V3_MODELS @@ -54,12 +58,17 @@ _MISTRAL_PARAMS: Final = frozenset( "table_format", "confidence_scores_granularity", "include_blocks", - "id", } ) _MISTRAL_2512_PARAMS: Final = _MISTRAL_PARAMS - {"include_blocks"} _MISTRAL_2505_PARAMS: Final = _MISTRAL_2512_PARAMS - {"extract_header", "extract_footer", "table_format"} +_AZURE_MISTRAL_PARAMS: Final = _MISTRAL_2505_PARAMS - {"document_annotation_prompt"} _FIND_SETTINGS: Final = settings(max_examples=2_000, deadline=None, derandomize=True, database=None) +_INLINE_IMAGE_DATA_URI: Final = "data:image/png;base64,dGVzdA==" + + +def discover_targets(environ: Mapping[str, str], client: OcrFixtureClient) -> tuple[OcrRecordingTarget, ...]: + return discover_targets_with_media(environ, client, _INLINE_IMAGE_DATA_URI) def _model(case_input: OcrSdkInputBase) -> str: @@ -75,6 +84,14 @@ def _find_input( return find(strategy, predicate, settings=_FIND_SETTINGS) +def _document_transport(case_input: OcrSdkInputBase) -> tuple[str, str]: + document: Final = cast(dict[str, object], case_input.canonical_input()["document"]) + document_type: Final = cast(str, document["type"]) + source: Final = document["image_url"] if document_type == "image_url" else document["document_url"] + assert isinstance(source, str) + return document_type, "data" if source.startswith("data:") else "remote" + + def test_parse_args_has_no_model_selection() -> None: args: Final = parse_recording_args(["--examples", "2", "--concurrency", "3", "--fixture-dir", "/tmp/ocr"]) @@ -191,7 +208,7 @@ def test_mistral_target_invocation_forwards_discovered_credentials() -> None: assert kwargs["model"] in MISTRAL_MODELS -def test_every_target_strategy_reaches_every_model_and_coverage_param() -> None: +def test_every_target_strategy_reaches_every_recording_model_and_coverage_param() -> None: targets: Final = discover_targets( { "MISTRAL_API_KEY": "mistral-secret", @@ -207,15 +224,15 @@ def test_every_target_strategy_reaches_every_model_and_coverage_param() -> None: ) expected: Final[dict[str, tuple[tuple[str, ...], frozenset[str]]]] = { "mistral-ocr": (MISTRAL_MODELS, _MISTRAL_PARAMS), - "azure-mistral": (AZURE_MISTRAL_MODELS, _MISTRAL_2512_PARAMS), + "azure-mistral": (AZURE_MISTRAL_MODELS, _AZURE_MISTRAL_PARAMS), "azure-document-intelligence": ( - AZURE_DOCUMENT_INTELLIGENCE_MODELS, + AZURE_DOCUMENT_INTELLIGENCE_RECORDING_MODELS, frozenset({"pages", "features", "req_format"}), ), "vertex-mistral": (VERTEX_MISTRAL_MODELS, _MISTRAL_2505_PARAMS), "vertex-deepseek": (VERTEX_DEEPSEEK_MODELS, frozenset[str]()), "reducto-v3": (REDUCTO_V3_MODELS, frozenset({"formatting", "retrieval", "settings"})), - "reducto-legacy": (REDUCTO_LEGACY_MODELS, frozenset({"enhance"})), + "reducto-legacy": (REDUCTO_LEGACY_MODELS, frozenset[str]()), } for target in targets: @@ -231,13 +248,57 @@ def test_every_target_strategy_reaches_every_model_and_coverage_param() -> None: == model ) for param in expected_params: - assert ( - param - in _find_input( - target.strategy, - lambda case_input, expected_param=param: expected_param in case_input.as_sdk_kwargs(), - ).as_sdk_kwargs() + reached = _find_input( + target.strategy, + lambda case_input, expected_param=param: expected_param in case_input.as_sdk_kwargs(), ) + assert param in reached.as_sdk_kwargs() + document = cast(dict[str, object], reached.canonical_input()["document"]) + assert document == {"type": "document_url", "document_url": structured_pdf_data_uri()} + + +@pytest.mark.parametrize("target_name", ("mistral-ocr", "azure-mistral", "vertex-mistral")) +def test_mistral_recording_targets_reach_every_transport_branch(target_name: str) -> None: + targets: Final = discover_targets( + { + "MISTRAL_API_KEY": "mistral-secret", + "AZURE_AI_API_KEY": "azure-secret", + "AZURE_AI_API_BASE": "https://azure.example", + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + target: Final = next(candidate for candidate in targets if candidate.name == target_name) + + for transport in ( + ("image_url", "remote"), + ("image_url", "data"), + ("document_url", "remote"), + ("document_url", "data"), + ): + reached = _find_input( + target.strategy, + lambda case_input, expected=transport: _document_transport(case_input) == expected, + ) + assert _document_transport(reached) == transport + + +def test_vertex_deepseek_recording_reaches_documented_image_branch() -> None: + targets: Final = discover_targets( + { + "VERTEX_AI_API_KEY": "vertex-secret", + "VERTEXAI_PROJECT": "project-1", + }, + _UNUSED_OCR_CLIENT, + ) + target: Final = next(candidate for candidate in targets if candidate.name == "vertex-deepseek") + case_input: Final = _find_input( + target.strategy, + lambda candidate: _document_transport(candidate) == ("image_url", "data"), + ) + + assert _document_transport(case_input) == ("image_url", "data") def test_ocr_targets_have_no_hardcoded_required_inputs() -> None: diff --git a/uv.lock b/uv.lock index 125640a112f..d5bb6475bac 100644 --- a/uv.lock +++ b/uv.lock @@ -4542,6 +4542,7 @@ dev = [ { name = "pytest-rerunfailures" }, { name = "pytest-timeout" }, { name = "pytest-xdist" }, + { name = "reportlab" }, { name = "requests-mock" }, { name = "responses" }, { name = "respx" }, @@ -4728,6 +4729,7 @@ dev = [ { name = "pytest-rerunfailures", specifier = "==15.1" }, { name = "pytest-timeout", specifier = "==2.4.0" }, { name = "pytest-xdist", specifier = "==3.8.0" }, + { name = "reportlab", specifier = "==5.0.1" }, { name = "requests-mock", specifier = "==1.12.1" }, { name = "responses", specifier = "==0.26.0" }, { name = "respx", specifier = "==0.22.0" }, @@ -8283,6 +8285,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/da/80/35b4c33c804a165a7f55289afda3ea9e3eb6d15800341a2d66455c0f1f30/regex-2026.5.9-cp314-cp314t-win_arm64.whl", hash = "sha256:5e41809d2683fcde7d5a8c87a6567ba1fb1ce0de9f31bff578de00a4b2d76daa", size = 275713, upload-time = "2026-05-09T23:15:16.98Z" }, ] +[[package]] +name = "reportlab" +version = "5.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "charset-normalizer" }, + { name = "pillow" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4a/51/dbe28534ae12c852f61be91f039f343305fd1f34f1c66b8de75afae7a525/reportlab-5.0.1.tar.gz", hash = "sha256:ebd13154be1c8515e665de70bd2d303ae9ddc3ef47e44afd5116441ca0283a26", size = 3945711, upload-time = "2026-08-20T13:48:16.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/db/cb/dacbc268cb68d0428ea2cbd85266195a9ab3e677449589ddae59bd7542ac/reportlab-5.0.1-py3-none-any.whl", hash = "sha256:1c36e6bb0e71780c72331eba60da7f602e8d4389a8723825af71342e49d791e8", size = 1957258, upload-time = "2026-08-20T13:48:14.026Z" }, +] + [[package]] name = "requests" version = "2.34.0"