From 41cc4b6020b0119889c33bb44300b5218fc3a637 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 4 Sep 2026 10:43:43 -0700 Subject: [PATCH] fix(ocr): honor configured download limit --- litellm-rust/crates/core/src/constants.rs | 1 - .../crates/core/src/ocr/common_utils.rs | 144 ++++++++++++++++-- litellm-rust/crates/core/src/ocr/handler.rs | 3 +- litellm-rust/crates/core/src/ocr/prepare.rs | 2 + litellm-rust/crates/core/src/ocr/tests.rs | 1 + litellm-rust/crates/core/src/ocr/types.rs | 2 + .../python-bridge/src/routes/definition.rs | 26 +++- .../crates/python-bridge/src/routes/ocr.rs | 2 + litellm/ocr/main.py | 3 +- litellm/rust_bridge/ocr.py | 7 + tests/test_litellm/ocr/test_rust_bridge.py | 28 ++++ .../rust_bridge/native_route_wheel_test.py | 1 + 12 files changed, 197 insertions(+), 23 deletions(-) diff --git a/litellm-rust/crates/core/src/constants.rs b/litellm-rust/crates/core/src/constants.rs index 4a920ca30df..e97fbc75313 100644 --- a/litellm-rust/crates/core/src/constants.rs +++ b/litellm-rust/crates/core/src/constants.rs @@ -33,7 +33,6 @@ pub(crate) const CHAT_COMPLETIONS_CONNECT_TIMEOUT_SECS: u64 = 10; pub(crate) const AUDIO_TRANSCRIPTION_TIMEOUT_SECS: u64 = 600; pub(crate) const OCR_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS: u64 = 120; -pub(crate) const OCR_DEFAULT_MAX_DOCUMENT_DOWNLOAD_BYTES: u64 = 50 * 1024 * 1024; pub(crate) const OCR_MAX_SAFE_FETCH_REDIRECTS: usize = 10; /// `object` field every non-streaming chat completion response carries. diff --git a/litellm-rust/crates/core/src/ocr/common_utils.rs b/litellm-rust/crates/core/src/ocr/common_utils.rs index 7f740c8a0ea..3fa4c0438aa 100644 --- a/litellm-rust/crates/core/src/ocr/common_utils.rs +++ b/litellm-rust/crates/core/src/ocr/common_utils.rs @@ -6,10 +6,7 @@ use base64::engine::general_purpose::STANDARD as BASE64_STANDARD; use reqwest::Url; use serde_json::{Map, Value}; -use crate::constants::{ - OCR_DEFAULT_MAX_DOCUMENT_DOWNLOAD_BYTES, OCR_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS, - OCR_MAX_SAFE_FETCH_REDIRECTS, -}; +use crate::constants::{OCR_DOCUMENT_INTELLIGENCE_POLL_TIMEOUT_SECS, OCR_MAX_SAFE_FETCH_REDIRECTS}; use crate::error::Error; use crate::http_utils::truncate_error_body; use crate::ocr::transformation::OcrProviderConfig; @@ -168,11 +165,8 @@ fn redirect_location(response: &reqwest::Response, url: &Url) -> Result Result<(Url, reqwest::Response), Error> { - let mut current_url = Url::parse(url) - .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?; - for _ in 0..OCR_MAX_SAFE_FETCH_REDIRECTS { validate_safe_fetch_url(¤t_url).await?; let response = client @@ -210,8 +204,8 @@ fn enforce_download_size(content_length: u64, max_bytes: u64, url: &Url) -> Resu async fn read_response_with_limit( mut response: reqwest::Response, url: &Url, + max_bytes: u64, ) -> Result, Error> { - let max_bytes = OCR_DEFAULT_MAX_DOCUMENT_DOWNLOAD_BYTES; if let Some(content_length) = response.content_length() { enforce_download_size(content_length, max_bytes, url)?; } else { @@ -235,6 +229,7 @@ async fn read_response_with_limit( pub(super) async fn convert_document_url_to_data_uri( client: &reqwest::Client, document: Value, + max_document_download_bytes: u64, ) -> Result { let Some((field, url)) = document_url_field(&document)? else { return Ok(document); @@ -243,7 +238,10 @@ pub(super) async fn convert_document_url_to_data_uri( return Ok(document); } - let (final_url, response) = safe_get_document_url(client, url).await?; + let initial_url = Url::parse(url) + .map_err(|err| Error::InvalidRequest(format!("invalid OCR document URL: {err}")))?; + enforce_download_size(0, max_document_download_bytes, &initial_url)?; + let (final_url, response) = safe_get_document_url(client, initial_url).await?; let status = response.status(); if !status.is_success() { let body = response.text().await.unwrap_or_default(); @@ -260,7 +258,7 @@ pub(super) async fn convert_document_url_to_data_uri( .filter(|value| !value.is_empty()) .unwrap_or("application/octet-stream") .to_string(); - let bytes = read_response_with_limit(response, &final_url).await?; + let bytes = read_response_with_limit(response, &final_url, max_document_download_bytes).await?; let data_uri = format!( "data:{content_type};base64,{}", BASE64_STANDARD.encode(bytes) @@ -381,9 +379,42 @@ pub(super) async fn poll_document_intelligence( mod tests { use crate::ocr::transformation::OcrResponseHandling; use serde_json::json; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; use super::*; + async fn response_with_content_length(content_length: u64) -> (Url, reqwest::Response) { + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("test listener binds"); + let address = listener.local_addr().expect("listener has local address"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let mut request = [0_u8; 1024]; + let bytes_read = socket.read(&mut request).await.expect("reads request"); + assert!(request[..bytes_read].starts_with(b"GET /document.pdf HTTP/1.1")); + socket + .write_all( + format!( + "HTTP/1.1 200 OK\r\ncontent-length: {content_length}\r\nconnection: close\r\n\r\n" + ) + .as_bytes(), + ) + .await + .expect("writes response"); + }); + let url = + Url::parse(&format!("http://{address}/document.pdf")).expect("document URL parses"); + let response = reqwest::Client::new() + .get(url.clone()) + .send() + .await + .expect("GET request succeeds"); + server.await.expect("server task completes"); + (url, response) + } + #[test] fn blocks_private_and_metadata_ips() { assert!(is_blocked_ip("127.0.0.1".parse().unwrap())); @@ -407,6 +438,7 @@ mod tests { "type": "image_url", "image_url": "http://127.0.0.1/image.png" }), + 50 * 1024 * 1024, ) .await .unwrap_err(); @@ -418,6 +450,25 @@ mod tests { )); } + #[tokio::test] + async fn zero_download_limit_blocks_before_url_fetch() { + let client = reqwest::Client::new(); + let error = convert_document_url_to_data_uri( + &client, + json!({ + "type": "image_url", + "image_url": "http://127.0.0.1/image.png" + }), + 0, + ) + .await + .unwrap_err(); + + assert!( + matches!(error, Error::InvalidRequest(message) if message.contains("download is disabled")) + ); + } + #[tokio::test] async fn convert_document_url_leaves_data_uri_untouched() { let client = reqwest::Client::new(); @@ -426,13 +477,78 @@ mod tests { "image_url": "data:image/png;base64,abcd" }); - let transformed = convert_document_url_to_data_uri(&client, document.clone()) - .await - .unwrap(); + let transformed = + convert_document_url_to_data_uri(&client, document.clone(), 50 * 1024 * 1024) + .await + .unwrap(); assert_eq!(transformed, document); } + #[tokio::test] + async fn configured_smaller_download_limit_is_honored() { + let (url, response) = response_with_content_length(1025).await; + + let error = read_response_with_limit(response, &url, 1024) + .await + .expect_err("configured limit blocks larger download"); + + assert!( + matches!(error, Error::InvalidRequest(message) if message.contains("exceeds maximum allowed size")) + ); + } + + #[tokio::test] + async fn configured_download_limit_above_default_is_honored() { + const CHUNK_SIZE_BYTES: usize = 64 * 1024; + const CHUNK_COUNT: usize = 800; + const RESPONSE_SIZE_BYTES: u64 = (CHUNK_SIZE_BYTES * CHUNK_COUNT + 1) as u64; + const CONFIGURED_LIMIT_BYTES: u64 = RESPONSE_SIZE_BYTES + 1; + + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("test listener binds"); + let address = listener.local_addr().expect("listener has local address"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let mut request = [0_u8; 1024]; + let bytes_read = socket.read(&mut request).await.expect("reads request"); + assert!(request[..bytes_read].starts_with(b"GET /document.pdf HTTP/1.1")); + socket + .write_all( + format!( + "HTTP/1.1 200 OK\r\ncontent-length: {RESPONSE_SIZE_BYTES}\r\nconnection: close\r\n\r\n" + ) + .as_bytes(), + ) + .await + .expect("writes response"); + let chunk = [0_u8; CHUNK_SIZE_BYTES]; + for _ in 0..CHUNK_COUNT { + socket.write_all(&chunk).await.expect("writes body chunk"); + } + socket + .write_all(&[0]) + .await + .expect("writes final body byte"); + }); + let url = + Url::parse(&format!("http://{address}/document.pdf")).expect("document URL parses"); + let response = reqwest::Client::new() + .get(url.clone()) + .send() + .await + .expect("GET request succeeds"); + assert_eq!(response.content_length(), Some(RESPONSE_SIZE_BYTES)); + + let bytes = read_response_with_limit(response, &url, CONFIGURED_LIMIT_BYTES) + .await + .expect("configured limit permits the advertised size"); + + assert_eq!(bytes.len() as u64, RESPONSE_SIZE_BYTES); + server.await.expect("server task completes"); + } + #[test] fn ocr_dispatch_supports_migrated_providers() { assert!(ocr_provider_config("mistral", "mistral-ocr-latest").is_some()); diff --git a/litellm-rust/crates/core/src/ocr/handler.rs b/litellm-rust/crates/core/src/ocr/handler.rs index a69fe656a6a..ad2f9a3b785 100644 --- a/litellm-rust/crates/core/src/ocr/handler.rs +++ b/litellm-rust/crates/core/src/ocr/handler.rs @@ -41,6 +41,7 @@ pub(crate) async fn execute_ocr_provider_call( optional_params, requires_reducto_upload, timeout, + max_document_download_bytes, } = request; let env_lookup = |key: &str| std::env::var(key).ok(); let upstream_headers = config.validate_environment( @@ -50,7 +51,7 @@ pub(crate) async fn execute_ocr_provider_call( )?; let url = config.complete_url(api_base.as_deref(), &model, &url_params, &env_lookup)?; let document = if config.requires_data_uri_document() { - convert_document_url_to_data_uri(client, document).await? + convert_document_url_to_data_uri(client, document, max_document_download_bytes).await? } else if requires_reducto_upload { upload_reducto_document( client, diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 91537711fd2..97e8d42e934 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -39,6 +39,7 @@ pub(super) async fn prepare_ocr_call(request: OcrRequest<'_>) -> Result(model: &'a str, api_base: &'a str) -> OcrRequest<'a> { extra_headers: None, optional_params: Map::new(), timeout: None, + max_document_download_bytes: 50 * 1024 * 1024, } } diff --git a/litellm-rust/crates/core/src/ocr/types.rs b/litellm-rust/crates/core/src/ocr/types.rs index ab59fefd9c7..bbf48c856cf 100644 --- a/litellm-rust/crates/core/src/ocr/types.rs +++ b/litellm-rust/crates/core/src/ocr/types.rs @@ -14,6 +14,7 @@ pub struct OcrRequest<'a> { pub extra_headers: Option>, pub optional_params: Map, pub timeout: Option, + pub max_document_download_bytes: u64, } pub(super) struct PreparedOcrRequest { @@ -27,6 +28,7 @@ pub(super) struct PreparedOcrRequest { pub(super) optional_params: Map, pub(super) requires_reducto_upload: bool, pub(super) timeout: Option, + pub(super) max_document_download_bytes: u64, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/litellm-rust/crates/python-bridge/src/routes/definition.rs b/litellm-rust/crates/python-bridge/src/routes/definition.rs index 3285da14d5f..6f74d5c857e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/definition.rs +++ b/litellm-rust/crates/python-bridge/src/routes/definition.rs @@ -225,7 +225,7 @@ mod tests { ( "ocr", "aocr", - "(model, document, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)", + "(model, document, max_document_download_bytes, api_key=None, api_base=None, custom_llm_provider=None, extra_headers=None, optional_params=None, timeout_seconds=None)", ), ( "transcription", @@ -305,13 +305,18 @@ mod tests { ); let invalid_headers = PyList::empty(py); - let kwargs = PyDict::new(py); - kwargs - .set_item("extra_headers", &invalid_headers) - .expect("kwargs should accept extra_headers"); let document = PyDict::new(py); for (sync_name, async_name) in [("ocr", "aocr"), ("transcription", "atranscription")] { + let kwargs = PyDict::new(py); + kwargs + .set_item("extra_headers", &invalid_headers) + .expect("kwargs should accept extra_headers"); + if sync_name == "ocr" { + kwargs + .set_item("max_document_download_bytes", 1024) + .expect("kwargs should accept max_document_download_bytes"); + } let sync_error = module .getattr(sync_name) .and_then(|function| function.call(("model", &document), Some(&kwargs))) @@ -378,10 +383,19 @@ mod tests { let invalid_payload = PyModule::new(py, "invalid_payload").expect("invalid payload should be created"); for name in ["ocr", "transcription"] { + let payload_kwargs = PyDict::new(py); + payload_kwargs + .set_item("extra_headers", &invalid) + .expect("kwargs should accept extra_headers"); + if name == "ocr" { + payload_kwargs + .set_item("max_document_download_bytes", 1024) + .expect("kwargs should accept max_document_download_bytes"); + } let error = module .getattr(name) .and_then(|function| { - function.call(("model", &invalid_payload), Some(&headers_kwargs)) + function.call(("model", &invalid_payload), Some(&payload_kwargs)) }) .expect_err("payload should be validated before headers"); assert!(!error.to_string().contains("extra_headers")); diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr.rs b/litellm-rust/crates/python-bridge/src/routes/ocr.rs index f6b928c4c9a..641ba36acfe 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr.rs @@ -109,6 +109,7 @@ fn prepare_ocr( extra_headers, optional_params, timeout, + max_document_download_bytes: inputs.max_document_download_bytes, }, ) .await @@ -245,6 +246,7 @@ bridge_route! { model: String, #[pyo3(from_py_with = litellm_python_interop::from_py)] document: serde_json::Value, + max_document_download_bytes: u64, }, optional = { api_key: Option, diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 0289a735054..777bb504e5c 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -16,7 +16,7 @@ import httpx import litellm from litellm._logging import verbose_logger -from litellm.constants import request_timeout +from litellm.constants import MAX_IMAGE_URL_DOWNLOAD_SIZE_MB, request_timeout from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.llms.azure_ai.ocr.common_utils import ( is_azure_document_intelligence_model, @@ -265,6 +265,7 @@ def _prepare_rust_ocr_call( optional_params=rust_optional_params, timeout=prepared_request.effective_timeout, logging_api_base=resolved_complete_url, + max_document_download_bytes=max(0, int(MAX_IMAGE_URL_DOWNLOAD_SIZE_MB * 1024 * 1024)), ) diff --git a/litellm/rust_bridge/ocr.py b/litellm/rust_bridge/ocr.py index cfa0664f330..851f3db010e 100644 --- a/litellm/rust_bridge/ocr.py +++ b/litellm/rust_bridge/ocr.py @@ -39,6 +39,7 @@ class RustOcr(Protocol): extra_headers: Mapping[str, object] | None, optional_params: Mapping[str, object], timeout_seconds: float | None, + max_document_download_bytes: int, ) -> Mapping[str, object]: raise NotImplementedError @@ -55,6 +56,7 @@ class RustAocr(Protocol): extra_headers: Mapping[str, object] | None, optional_params: Mapping[str, object], timeout_seconds: float | None, + max_document_download_bytes: int, ) -> Awaitable[Mapping[str, object]]: raise NotImplementedError @@ -125,6 +127,7 @@ class RustOCRRequest: optional_params: Mapping[str, object] timeout: float | httpx.Timeout | None logging_api_base: str | None = None + max_document_download_bytes: int = 0 @dataclass(frozen=True, slots=True) @@ -324,6 +327,7 @@ def _call_ocr(rust_ocr: RustOcr, request: RustOCRRequest) -> Mapping[str, object extra_headers=request.extra_headers, optional_params=request.optional_params, timeout_seconds=_timeout_to_seconds(request.timeout), + max_document_download_bytes=request.max_document_download_bytes, ) @@ -337,6 +341,7 @@ def _call_aocr(rust_aocr: RustAocr, request: RustOCRRequest) -> Awaitable[Mappin extra_headers=request.extra_headers, optional_params=request.optional_params, timeout_seconds=_timeout_to_seconds(request.timeout), + max_document_download_bytes=request.max_document_download_bytes, ) @@ -350,6 +355,7 @@ def _call_prepare(rust_prepare: RustOcrPrepare, request: RustOCRRequest) -> obje extra_headers=request.extra_headers, optional_params=request.optional_params, timeout_seconds=_timeout_to_seconds(request.timeout), + max_document_download_bytes=request.max_document_download_bytes, ) @@ -363,4 +369,5 @@ def _call_aprepare(rust_prepare: RustAocrPrepare, request: RustOCRRequest) -> Aw extra_headers=request.extra_headers, optional_params=request.optional_params, timeout_seconds=_timeout_to_seconds(request.timeout), + max_document_download_bytes=request.max_document_download_bytes, ) diff --git a/tests/test_litellm/ocr/test_rust_bridge.py b/tests/test_litellm/ocr/test_rust_bridge.py index 79b80d103b6..14b5294985e 100644 --- a/tests/test_litellm/ocr/test_rust_bridge.py +++ b/tests/test_litellm/ocr/test_rust_bridge.py @@ -77,6 +77,7 @@ class RecordingBridge: extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout_seconds: float | None, + max_document_download_bytes: int, ) -> dict[str, object]: self.calls.append( { @@ -88,6 +89,7 @@ class RecordingBridge: "extra_headers": extra_headers, "optional_params": optional_params, "timeout_seconds": timeout_seconds, + "max_document_download_bytes": max_document_download_bytes, } ) return dict(FAKE_OCR_RESPONSE) @@ -109,6 +111,7 @@ class RecordingAsyncBridge: extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout_seconds: float | None, + max_document_download_bytes: int, ) -> dict[str, object]: self.calls.append( { @@ -120,6 +123,7 @@ class RecordingAsyncBridge: "extra_headers": extra_headers, "optional_params": optional_params, "timeout_seconds": timeout_seconds, + "max_document_download_bytes": max_document_download_bytes, } ) return dict(FAKE_OCR_RESPONSE) @@ -136,6 +140,7 @@ class RaisingBridge: extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout_seconds: float | None, + max_document_download_bytes: int, ) -> dict[str, object]: raise RuntimeError("bridge failed") @@ -151,6 +156,7 @@ class RaisingAsyncBridge: extra_headers: dict[str, object] | None, optional_params: dict[str, object], timeout_seconds: float | None, + max_document_download_bytes: int, ) -> dict[str, object]: raise RuntimeError("bridge failed") @@ -467,6 +473,7 @@ def test_bridge_wrapper_forwards_prepared_args_and_wraps_response(): extra_headers={"Authorization": "Bearer sk-test", "x-trace-id": "trace-1"}, optional_params={"include_image_base64": True, "pages": [0]}, timeout=12.5, + max_document_download_bytes=1024, ), on_accepted=lambda _request: None, ) @@ -485,6 +492,7 @@ def test_bridge_wrapper_forwards_prepared_args_and_wraps_response(): }, "optional_params": {"include_image_base64": True, "pages": [0]}, "timeout_seconds": 12.5, + "max_document_download_bytes": 1024, } @@ -510,6 +518,7 @@ async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response(): extra_headers=None, optional_params={"vertex_project": "project-1"}, timeout=httpx.Timeout(30.0, read=42.0), + max_document_download_bytes=2048, ), on_accepted=lambda _request: None, ) @@ -524,6 +533,7 @@ async def test_bridge_wrapper_forwards_prepared_async_args_and_wraps_response(): "extra_headers": None, "optional_params": {"vertex_project": "project-1"}, "timeout_seconds": 42.0, + "max_document_download_bytes": 2048, } @@ -559,6 +569,7 @@ def test_run_rust_ocr_prepares_request_and_wraps_response(): }, "optional_params": {"include_image_base64": True}, "timeout_seconds": 12.5, + "max_document_download_bytes": 50 * 1024 * 1024, } @@ -902,6 +913,23 @@ def test_ocr_passes_default_request_timeout_to_rust(fake_bridge): assert fake_bridge.calls[0]["timeout_seconds"] == float(request_timeout) +@pytest.mark.parametrize( + ("configured_mb", "expected_bytes"), + [(0.0, 0), (0.25, 256 * 1024), (-1.0, 0)], +) +def test_ocr_passes_configured_download_limit_to_rust( + monkeypatch: pytest.MonkeyPatch, + fake_bridge: RecordingBridge, + configured_mb: float, + expected_bytes: int, +) -> None: + monkeypatch.setattr(ocr_main, "MAX_IMAGE_URL_DOWNLOAD_SIZE_MB", configured_mb) + + litellm.ocr(model=MODEL, document=DOCUMENT, api_key="sk-test") + + assert fake_bridge.calls[0]["max_document_download_bytes"] == expected_bytes + + @pytest.mark.parametrize("disable_source", ("default", "environment", "process", "request")) def test_ocr_disabled_never_loads_or_prepares_native_bridge( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/test_litellm/rust_bridge/native_route_wheel_test.py b/tests/test_litellm/rust_bridge/native_route_wheel_test.py index a7f50a82a99..6399258cd06 100644 --- a/tests/test_litellm/rust_bridge/native_route_wheel_test.py +++ b/tests/test_litellm/rust_bridge/native_route_wheel_test.py @@ -136,6 +136,7 @@ def route_kwargs(route: str, api_base: str, outcome: str) -> dict[str, object]: "api_key": "sk-native", "custom_llm_provider": "mistral", "optional_params": {"include_image_base64": True}, + "max_document_download_bytes": 50 * 1024 * 1024, } if route == "transcription": return common | {