From 566cf2d3d7f076f293ed6c8ee01232c8589ce583 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Sat, 19 Sep 2026 09:55:00 -0700 Subject: [PATCH] wip --- litellm-rust/Cargo.lock | 2 + litellm-rust/Cargo.toml | 1 + .../crates/callbacks-legacy/src/adapter.rs | 14 ++++ .../crates/callbacks-legacy/src/callbacks.rs | 54 +++++++++++++ .../crates/callbacks-legacy/tests/support.rs | 1 + litellm-rust/crates/core-utils/Cargo.toml | 1 + .../crates/core-utils/src/url_utils.rs | 67 ++++++++++++++++ .../crates/core/src/ocr/provider_config.rs | 18 +++-- litellm-rust/crates/core/src/ocr/wire.rs | 23 ++++-- litellm-rust/crates/llms/Cargo.toml | 1 + .../document_intelligence/transformation.rs | 79 ++++++++++++++++--- .../crates/llms/src/base_llm/ocr/document.rs | 4 +- .../crates/llms/src/base_llm/ocr/error.rs | 11 +++ .../llms/src/base_llm/ocr/transformation.rs | 46 +++++++++-- .../python-bridge/src/routes/messages/mod.rs | 1 + .../python-bridge/src/routes/ocr/errors.rs | 8 ++ .../python-bridge/src/routes/ocr/mod.rs | 1 + .../python-bridge/src/routes/ocr/project.rs | 56 +++++++------ litellm/ocr/main.py | 12 +-- litellm/rust_bridge/failures.py | 14 ++-- litellm/rust_bridge/ocr/route_host.py | 29 ++++++- .../shared/parity/runner.py | 20 +++-- .../e2e_parity/sdk/ocr/test_sdk_parity.py | 36 ++++----- .../rust_bridge/ocr/test_route_host.py | 30 +++++++ .../test_litellm/rust_bridge/test_failures.py | 15 ++++ tests/test_litellm/rust_bridge/test_loader.py | 27 +++++++ 26 files changed, 476 insertions(+), 95 deletions(-) create mode 100644 tests/test_litellm/rust_bridge/test_loader.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 860f01c4ad1..5db918851fb 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -2085,6 +2085,7 @@ version = "0.1.0" dependencies = [ "fancy-regex", "litellm-types", + "percent-encoding", "rstest", "serde", "serde_json", @@ -2170,6 +2171,7 @@ dependencies = [ "litellm-host", "litellm-http", "litellm-types", + "percent-encoding", "reqwest 0.12.28", "rstest", "serde", diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index 8634dce92d0..03877640793 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -52,6 +52,7 @@ base64 = "0.22" moka = { version = "0.12.16", features = ["future"] } strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" +percent-encoding = "2.3.2" webpki-roots = "1" time = { version = "0.3.53", features = ["parsing"] } criterion = "0.8.2" diff --git a/litellm-rust/crates/callbacks-legacy/src/adapter.rs b/litellm-rust/crates/callbacks-legacy/src/adapter.rs index 6c013cd1ea5..ebaa17c20ce 100644 --- a/litellm-rust/crates/callbacks-legacy/src/adapter.rs +++ b/litellm-rust/crates/callbacks-legacy/src/adapter.rs @@ -32,6 +32,10 @@ pub struct LegacySurface { pub input_description: &'static str, /// How a streamed response is billed; `None` for a route that never streams. pub stream: Option, + /// Whether `Logging` learns the model, provider and metadata before provider + /// preparation, as the Python route does, so a failure ahead of the wire request + /// (credentials, document download) is still attributed. + pub updates_logging_before_preparation: bool, } /// The pass-through billing a streamed response goes through once its chunks are in. @@ -123,6 +127,16 @@ impl LegacyLogging { fn prepare(&mut self, py: Python<'_>) -> PyResult { let prepared = prepare(py, self.call.kwargs().bind(py), self.logger()?)?.unbind(); self.call.set_kwargs(prepared); + if self.surface.updates_logging_before_preparation { + let model = self.call.lookup(py, "model")?; + let provider = self.call.lookup(py, "custom_llm_provider")?; + self.logger()?.update_before_preparation( + py, + self.call.kwargs(), + model.and_then(|model| model.extract::().ok()), + provider.and_then(|provider| provider.extract::().ok()), + )?; + } Ok(LifecycleStep::Arguments(self.call.kwargs().clone_ref(py))) } diff --git a/litellm-rust/crates/callbacks-legacy/src/callbacks.rs b/litellm-rust/crates/callbacks-legacy/src/callbacks.rs index 5f04224e6d7..9a197e6ca67 100644 --- a/litellm-rust/crates/callbacks-legacy/src/callbacks.rs +++ b/litellm-rust/crates/callbacks-legacy/src/callbacks.rs @@ -229,6 +229,60 @@ impl LegacyCallbacks for PythonLogger { } } +impl PythonLogger { + /// `Logging.update_from_kwargs` with what the caller's arguments alone say, ahead of + /// provider preparation. Only the metadata keys are handed over, so no credential in + /// the keyword view reaches the logger unredacted. + pub(crate) fn update_before_preparation( + &self, + py: Python<'_>, + kwargs: &Py, + model: Option, + custom_llm_provider: Option, + ) -> PyResult<()> { + let Some(model) = model else { + return Ok(()); + }; + let (provider, model) = match custom_llm_provider.filter(|provider| !provider.is_empty()) { + Some(provider) => { + let model = model + .strip_prefix(provider.as_str()) + .and_then(|model| model.strip_prefix('/')) + .unwrap_or(&model) + .to_string(); + (provider, model) + } + None => match model.split_once('/') { + Some((provider, model)) if !provider.is_empty() && !model.is_empty() => { + (provider.to_string(), model.to_string()) + } + _ => return Ok(()), + }, + }; + let kwargs = kwargs.bind(py); + let metadata = PyDict::new(py); + for name in ["metadata", "litellm_metadata"] { + if let Some(value) = kwargs.get_item(name)? { + metadata.set_item(name, value)?; + } + } + let params = PyDict::new(py); + params.set_item("litellm_call_id", kwargs.get_item("litellm_call_id")?)?; + Logging::Update.call( + py, + ( + self.object(py), + metadata, + model, + PyDict::new(py), + params, + provider, + ), + )?; + Ok(()) + } +} + fn custom_pricing_fields(py: Python<'_>) -> PyResult> { Logging::CustomPricingFields.call(py, ())?.extract() } diff --git a/litellm-rust/crates/callbacks-legacy/tests/support.rs b/litellm-rust/crates/callbacks-legacy/tests/support.rs index d3cc32e301f..c118f0505e9 100644 --- a/litellm-rust/crates/callbacks-legacy/tests/support.rs +++ b/litellm-rust/crates/callbacks-legacy/tests/support.rs @@ -198,6 +198,7 @@ pub(crate) fn legacy_call( call_type: "test", input_description: "test input", stream: None, + updates_logging_before_preparation: false, }, call, asynchronous, diff --git a/litellm-rust/crates/core-utils/Cargo.toml b/litellm-rust/crates/core-utils/Cargo.toml index eb353bc060c..0ee47b36c1c 100644 --- a/litellm-rust/crates/core-utils/Cargo.toml +++ b/litellm-rust/crates/core-utils/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [dependencies] fancy-regex.workspace = true litellm-types.workspace = true +percent-encoding.workspace = true serde.workspace = true serde_json.workspace = true serde_path_to_error = "0.1" diff --git a/litellm-rust/crates/core-utils/src/url_utils.rs b/litellm-rust/crates/core-utils/src/url_utils.rs index 1f690752c7a..77d959b1185 100644 --- a/litellm-rust/crates/core-utils/src/url_utils.rs +++ b/litellm-rust/crates/core-utils/src/url_utils.rs @@ -1,5 +1,6 @@ use std::marker::PhantomData; +use percent_encoding::{AsciiSet, utf8_percent_encode}; use url::Url; #[derive(Debug, thiserror::Error)] @@ -64,6 +65,35 @@ impl ApiUrl { self } + /// Like [`Self::append_query_pairs`], but percent-encodes keys and values with `escape` + /// instead of form encoding, so callers choose which reserved characters stay literal. + /// An existing query is kept; an empty result leaves no dangling `?`. + pub fn append_query_pairs_escaped<'a>( + mut self, + pairs: impl IntoIterator, + escape: &'static AsciiSet, + ) -> Self { + let appended = pairs.into_iter().map(|(key, value)| { + format!( + "{}={}", + utf8_percent_encode(key, escape), + utf8_percent_encode(value, escape) + ) + }); + let query = self + .url + .query() + .filter(|existing| !existing.is_empty()) + .map(str::to_owned) + .into_iter() + .chain(appended) + .collect::>() + .join("&"); + self.url + .set_query(Some(query.as_str()).filter(|query| !query.is_empty())); + self + } + pub fn into_string(self) -> String { self.url.into() } @@ -111,4 +141,41 @@ mod tests { "https://example.test/analyze?model=name+with+spaces" ); } + + const KEEP_COMMAS: &AsciiSet = &percent_encoding::NON_ALPHANUMERIC.remove(b','); + + #[rstest::rstest] + #[case::fresh_query("https://example.test", &[("pages", "1,2")], "?pages=1,2")] + #[case::existing_query_kept( + "https://example.test?tenant=a", + &[("pages", "1,2")], + "?tenant=a&pages=1,2" + )] + #[case::bare_question_mark("https://example.test?", &[("pages", "1")], "?pages=1")] + #[case::no_pairs_no_query("https://example.test", &[], "")] + #[case::no_pairs_keeps_query("https://example.test?tenant=a", &[], "?tenant=a")] + #[case::outside_set_escaped( + "https://example.test", + &[("q", "a b&c=d+e%2C#")], + "?q=a%20b%26c%3Dd%2Be%252C%23" + )] + #[case::keys_escaped_too("https://example.test", &[("a&b", "1")], "?a%26b=1")] + #[case::non_ascii_as_utf8("https://example.test", &[("q", "é")], "?q=%C3%A9")] + fn escaped_query_pairs( + #[case] base: &str, + #[case] pairs: &[(&str, &str)], + #[case] expected_query: &str, + ) { + let actual = ApiUrl::parse(base) + .and_then(|url| url.complete_path(&["analyze"])) + .map(|url| { + url.append_query_pairs_escaped(pairs.iter().copied(), KEEP_COMMAS) + .into_string() + }) + .expect("url builds"); + assert_eq!( + actual, + format!("https://example.test/analyze{expected_query}") + ); + } } diff --git a/litellm-rust/crates/core/src/ocr/provider_config.rs b/litellm-rust/crates/core/src/ocr/provider_config.rs index d38d87b92cc..ab3a59669b6 100644 --- a/litellm-rust/crates/core/src/ocr/provider_config.rs +++ b/litellm-rust/crates/core/src/ocr/provider_config.rs @@ -1,4 +1,4 @@ -use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; +use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; use litellm_llms::{ aws_textract::ocr::{ analyze_transformation::TextractAnalyzeDocumentConfig, common_utils::TextractOperation, @@ -168,11 +168,8 @@ pub(crate) fn resolve_provider_config( model: &str, custom_llm_provider: Option<&str>, ) -> Result<(String, OcrConfigKind), Error> { - let provider = - get_custom_llm_provider(model, custom_llm_provider).unwrap_or(CustomLlmProvider { - model, - custom_llm_provider: OcrProvider::Mistral.into(), - }); + let provider = get_custom_llm_provider(model, custom_llm_provider) + .ok_or_else(|| Error::ProviderNotProvided(model.to_string()))?; let ocr_provider = provider .custom_llm_provider .parse::() @@ -482,6 +479,15 @@ mod tests { ); } + #[rstest] + #[case::bare("mistral-ocr-latest")] + #[case::other_route_model("gpt-4o")] + fn ocr_contract_model_without_a_provider_is_rejected_like_python(#[case] model: &str) { + let error = resolve_provider_config(model, None).unwrap_err(); + assert!(matches!(&error, Error::ProviderNotProvided(passed) if passed == model)); + assert_eq!(error.http_status_code(), Some(400)); + } + #[rstest] #[case::prefix("not_a_provider/model", None)] #[case::explicit("model", Some("not_a_provider"))] diff --git a/litellm-rust/crates/core/src/ocr/wire.rs b/litellm-rust/crates/core/src/ocr/wire.rs index b9c60f57e3c..99210e210fb 100644 --- a/litellm-rust/crates/core/src/ocr/wire.rs +++ b/litellm-rust/crates/core/src/ocr/wire.rs @@ -96,7 +96,17 @@ pub fn decode_request_input>( } pub fn decode_document(value: Value) -> Result { - let kind = value.get("type").and_then(Value::as_str); + let Some(fields) = value.as_object() else { + return Err(Error::DocumentNotObject); + }; + let kind = match fields.get("type") { + Some(Value::String(kind)) if matches!(kind.as_str(), "document_url" | "image_url") => { + Some(kind.as_str()) + } + Some(Value::String(kind)) => return Err(Error::InvalidDocumentType(kind.clone())), + Some(other) => return Err(Error::InvalidDocumentType(other.to_string())), + None => return Err(Error::InvalidDocumentType("None".into())), + }; if matches!(kind, Some("document_url")) && value.get("document_url").is_none() || matches!(kind, Some("image_url")) && value.get("image_url").is_none() { @@ -122,9 +132,12 @@ mod tests { } #[rstest] - #[case::non_object(json!([]), "document")] - #[case::missing_type(json!({"document_url":"https://example.com/a.pdf"}), "document")] - #[case::unsupported_type(json!({"type":"text"}), "type")] + #[case::non_object(json!([]), "document must be a dict")] + #[case::missing_type( + json!({"document_url":"https://example.com/a.pdf"}), + "Invalid document type: None" + )] + #[case::unsupported_type(json!({"type":"text"}), "Invalid document type: text")] #[case::missing_document_url(json!({"type":"document_url"}), "Document URL")] #[case::missing_image_url(json!({"type":"image_url"}), "Document URL")] fn ocr_contract_malformed_document_is_bad_request( @@ -134,7 +147,7 @@ mod tests { let error = decode_document(document).unwrap_err(); assert!(matches!( error, - Error::RequestField { .. } | Error::MissingDocumentUrl + Error::DocumentNotObject | Error::InvalidDocumentType(_) | Error::MissingDocumentUrl )); assert_eq!(error.http_status_code(), Some(400)); assert!(error.to_string().contains(field), "{error}"); diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 0cc7af1836f..efece8114af 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -22,6 +22,7 @@ base64.workspace = true bytes.workspace = true data-url = "0.3.2" futures-util.workspace = true +percent-encoding.workspace = true reqwest.workspace = true serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } diff --git a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs index 9b27fdbb568..e6a3288fe44 100644 --- a/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/ocr/document_intelligence/transformation.rs @@ -8,6 +8,7 @@ use litellm_core_utils::{ serde_compat::{FiniteF64, LaxI64}, url_utils::ApiUrl, }; +use percent_encoding::{AsciiSet, NON_ALPHANUMERIC}; use reqwest::Url; use serde::{Deserialize, Deserializer, Serialize}; use serde_json::{Map, Value}; @@ -549,6 +550,17 @@ async fn poll_operation( } } +/// Python parity: the Python provider builds this query with +/// `urllib.parse.quote(value, safe=",")`, which escapes everything except RFC 3986 unreserved +/// characters and `,`. The literal commas are also what the service documents for the +/// `pages` and `features` lists; form encoding would send `%2C`. +const PYTHON_QUOTE_KEEP_COMMAS: &AsciiSet = &NON_ALPHANUMERIC + .remove(b'-') + .remove(b'.') + .remove(b'_') + .remove(b'~') + .remove(b','); + impl AzureDocumentIntelligenceOcrConfig { fn build_ocr_url( &self, @@ -561,16 +573,15 @@ impl AzureDocumentIntelligenceOcrConfig { ApiUrl::parse(endpoint) .and_then(|url| url.complete_path(&["documentintelligence", "documentModels", &model])) .map(|url| { - url.append_query_pairs( - [("api-version", api_version)] - .into_iter() - .chain(params.pages.iter().map(|pages| ("pages", pages.as_str()))) - .chain( - params - .features - .iter() - .map(|features| ("features", features.as_str())), - ), + url.append_query_pairs_escaped( + [ + ("api-version", Some(api_version)), + ("pages", params.pages.as_deref()), + ("features", params.features.as_deref()), + ] + .into_iter() + .filter_map(|(key, value)| Some((key, value?))), + PYTHON_QUOTE_KEEP_COMMAS, ) .into_string() }) @@ -649,6 +660,54 @@ mod tests { use super::*; + #[test] + fn query_escape_set_matches_python_quote() { + // Expected output from Python: + // quote("".join(map(chr, range(0x20, 0x7f))) + "é", safe=",") + let input = (0x20u8..0x7f) + .map(char::from) + .chain(['é']) + .collect::(); + assert_eq!( + percent_encoding::utf8_percent_encode(&input, PYTHON_QUOTE_KEEP_COMMAS).to_string(), + "%20%21%22%23%24%25%26%27%28%29%2A%2B,-.%2F0123456789%3A%3B%3C%3D%3E%3F%40\ + ABCDEFGHIJKLMNOPQRSTUVWXYZ%5B%5C%5D%5E_%60abcdefghijklmnopqrstuvwxyz%7B%7C%7D~%C3%A9" + ); + } + + #[rstest] + #[case::no_lists(None, None, "")] + #[case::pages_only(Some("1,3-4"), None, "&pages=1,3-4")] + #[case::features_only( + None, + Some("keyValuePairs,languages"), + "&features=keyValuePairs,languages" + )] + #[case::both_lists( + Some("1,3-4"), + Some("keyValuePairs,languages"), + "&pages=1,3-4&features=keyValuePairs,languages" + )] + fn ocr_url_query_matches_python( + #[case] pages: Option<&str>, + #[case] features: Option<&str>, + #[case] expected_lists: &str, + ) { + let params = DocumentIntelligenceParams { + pages: pages.map(Into::into), + features: features.map(Into::into), + }; + assert_eq!( + AzureDocumentIntelligenceOcrConfig + .build_ocr_url("https://di.test", "prebuilt-layout", ¶ms, "2024-11-30") + .unwrap(), + format!( + "https://di.test/documentintelligence/documentModels/prebuilt-layout:analyze\ + ?api-version=2024-11-30{expected_lists}" + ) + ); + } + fn map(value: Value) -> Result { let arguments = serde_json::from_value(value).unwrap(); AzureDocumentIntelligenceOcrConfig.map_ocr_params(&arguments, "model") diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/document.rs b/litellm-rust/crates/llms/src/base_llm/ocr/document.rs index 724625b8208..b950033512b 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/document.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/document.rs @@ -198,7 +198,7 @@ mod tests { client.document_fetcher(), OcrDocument::ImageUrl { image_url: format!("http://{address}/image"), - extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]), + extra_fields: Map::from_iter([("detail".into(), "high".into())]), }, &OcrConnection::default(), ) @@ -210,7 +210,7 @@ mod tests { converted, OcrDocument::ImageUrl { image_url: "data:image/png;base64,YWJj".into(), - extra_fields: Map::from_iter([("detail".into(), Some("high".into()))]), + extra_fields: Map::from_iter([("detail".into(), "high".into())]), } ); assert!(!request.to_ascii_lowercase().contains("authorization")); diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs index e09842e2856..67345aecb19 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs @@ -30,6 +30,10 @@ pub enum Error { MissingField(&'static str), #[error("Document URL is required")] MissingDocumentUrl, + #[error("document must be a dict with 'type' and URL/file field")] + DocumentNotObject, + #[error("Invalid document type: {0}. Must be 'document_url', 'image_url', or 'file'")] + InvalidDocumentType(String), #[error("invalid OCR document data URI")] InvalidDataUri, #[error( @@ -76,6 +80,10 @@ pub enum Error { Unsupported(&'static str), #[error("invalid provider: {0}")] InvalidProvider(String), + #[error( + "LLM Provider NOT provided. Pass in the LLM provider you are trying to call. You passed model={0}" + )] + ProviderNotProvided(String), #[error("invalid model: {provider} has no model {model:?} - use one of: {}", supported.join(", "))] InvalidModel { provider: &'static str, @@ -149,6 +157,8 @@ impl Error { | Self::RequestField { .. } | Self::MissingField(_) | Self::MissingDocumentUrl + | Self::DocumentNotObject + | Self::InvalidDocumentType(_) | Self::InvalidDataUri | Self::ReductoSource | Self::InlineDocumentTooLarge @@ -161,6 +171,7 @@ impl Error { | Self::DotModel | Self::InvalidRequest(_) | Self::InvalidProvider(_) + | Self::ProviderNotProvided(_) | Self::InvalidModel { .. } | Self::Params(_) | Self::Headers(_) diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs index e02a4b7f266..9c075b20c23 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/transformation.rs @@ -32,16 +32,31 @@ pub enum OcrDocument { DocumentUrl { document_url: String, #[serde(flatten)] - extra_fields: BTreeMap>, + extra_fields: BTreeMap, }, #[serde(rename = "image_url")] ImageUrl { + #[serde(deserialize_with = "image_url_source")] image_url: String, #[serde(flatten)] - extra_fields: BTreeMap>, + extra_fields: BTreeMap, }, } +/// Mistral accepts `image_url` as a string or as `{"url": ...}`; providers only consume the +/// source, so the object form is read down to its URL. +fn image_url_source<'de, D: serde::Deserializer<'de>>(deserializer: D) -> Result { + #[derive(Deserialize)] + #[serde(untagged)] + enum ImageUrl { + Source(String), + Object { url: String }, + } + Ok(match ImageUrl::deserialize(deserializer)? { + ImageUrl::Source(url) | ImageUrl::Object { url } => url, + }) +} + impl OcrDocument { pub fn source(&self) -> &str { match self { @@ -346,8 +361,13 @@ pub struct DecodedOcrResponse { pub fn decode_request_value(value: Value, prefix: &str) -> Result { serde_path_to_error::deserialize(value.into_deserializer()).map_err(|error| { + let path = error.path().to_string(); Error::RequestField { - path: format!("{prefix}.{}", error.path()), + path: if path == "." { + prefix.to_string() + } else { + format!("{prefix}.{path}") + }, } }) } @@ -639,10 +659,8 @@ mod tests { assert!(serde_json::from_value::(Value::Object(payload)).is_err()); } assert!( - serde_json::from_value::(json!({ - "type":"image_url", "image_url":"https://example.com/image", "detail":42 - })) - .is_err() + serde_json::from_value::(json!({"type":"image_url", "image_url":42})) + .is_err() ); } @@ -694,6 +712,20 @@ mod tests { ); } + #[test] + fn documents_accept_what_python_forwards_to_the_provider() { + let document: OcrDocument = serde_json::from_value(json!({ + "type":"image_url", "image_url":{"url":"https://example.com/image", "detail":"high"} + })) + .unwrap(); + assert_eq!(document.source(), "https://example.com/image"); + let document = json!({ + "type":"document_url", "document_url":"https://example.com/a.pdf", "n":3 + }); + let decoded: OcrDocument = serde_json::from_value(document.clone()).unwrap(); + assert_eq!(serde_json::to_value(decoded).unwrap(), document); + } + #[test] fn response_serialization_flattens_extra_fields_and_omits_absent_native_response() { let response = LiteLLMOcrResponse { diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index 8c42315ac59..9bec0a756c0 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -17,6 +17,7 @@ const SURFACE: LegacySurface = LegacySurface { url_route: "/v1/messages", endpoint_type: "anthropic", }), + updates_logging_before_preparation: false, }; fn run_messages( diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs index b0a6acdebfd..6111506f4e5 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/errors.rs @@ -26,6 +26,14 @@ pub(super) fn to_pyerr(error: Error) -> PyErr { .ok(); error } + Error::InvalidProvider(provider) => { + let error = core_error_to_pyerr(Error::InvalidProvider(provider.clone()).into()); + error + .value(py) + .setattr("ocr_invalid_provider", provider) + .ok(); + error + } Error::FileRead { path, source } if source.kind() == std::io::ErrorKind::NotFound => { PyFileNotFoundError::new_err(format!("File not found: {}", path.display())) } diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs index 9be3171f70b..ef9ef523e33 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/mod.rs @@ -25,6 +25,7 @@ const SURFACE: LegacySurface = LegacySurface { call_type: "ocr", input_description: "OCR document processing", stream: None, + updates_logging_before_preparation: true, }; const ASYNC_SURFACE: LegacySurface = LegacySurface { diff --git a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs index 697b935a1d4..65012622781 100644 --- a/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs +++ b/litellm-rust/crates/python-bridge/src/routes/ocr/project.rs @@ -83,21 +83,27 @@ enum ProjectedDocument { impl ProjectedDocument { fn project(document: &Bound<'_, PyAny>) -> PyResult { - let kind: String = document - .get_item("type") - .and_then(|value| value.extract()) - .map_err(|error| { - let py = document.py(); - if error.is_instance_of::(py) - || error.is_instance_of::(py) - { - ocr_error_to_pyerr(Error::RequestField { - path: "document.type".into(), - }) - } else { - error - } - })?; + let py = document.py(); + if !document.is_instance_of::() && !document.hasattr("__getitem__")? { + return Err(ocr_error_to_pyerr(Error::DocumentNotObject)); + } + let kind = match document.get_item("type") { + Ok(value) => value, + Err(error) if error.is_instance_of::(py) => { + return Err(ocr_error_to_pyerr(Error::InvalidDocumentType( + "None".into(), + ))); + } + Err(error) if error.is_instance_of::(py) => { + return Err(ocr_error_to_pyerr(Error::DocumentNotObject)); + } + Err(error) => return Err(error), + }; + let Ok(kind) = kind.extract::() else { + return Err(ocr_error_to_pyerr(Error::InvalidDocumentType( + kind.str()?.to_string(), + ))); + }; if kind != "file" { return Ok(Self::Other(from_py(document)?)); } @@ -471,21 +477,25 @@ document = Document() } #[rstest::rstest] - #[case::missing(c"{}")] - #[case::non_string(c"{'type': 1}")] - #[case::list(c"[]")] - fn malformed_document_discriminators_are_bad_requests_naming_the_field( + #[case::missing( + c"{}", + "Invalid document type: None. Must be 'document_url', 'image_url', or 'file'" + )] + #[case::non_string( + c"{'type': 1}", + "Invalid document type: 1. Must be 'document_url', 'image_url', or 'file'" + )] + #[case::list(c"[]", "document must be a dict with 'type' and URL/file field")] + fn malformed_document_discriminators_are_bad_requests_with_the_python_message( #[case] document: &std::ffi::CStr, + #[case] expected: &str, ) { Python::initialize(); Python::attach(|py| { let error = project_document(&py.eval(document, None, None).unwrap()).unwrap_err(); let value = error.value(py); assert!(error.is_instance_of::(py)); - assert_eq!( - value.to_string(), - "invalid OCR request field: document.type" - ); + assert_eq!(value.to_string(), expected); assert_eq!( value .getattr("status_code") diff --git a/litellm/ocr/main.py b/litellm/ocr/main.py index 06830ed4b53..12ff897f4bb 100644 --- a/litellm/ocr/main.py +++ b/litellm/ocr/main.py @@ -73,7 +73,7 @@ def _prepare_ocr_request( raise litellm.BadRequestError( message="document must be a dict with 'type' and URL/file field", model=model, - llm_provider=_error_provider(model, custom_llm_provider) or "", + llm_provider=resolve_error_provider(model, custom_llm_provider) or "", ) normalized_document: Final = ( @@ -85,13 +85,13 @@ def _prepare_ocr_request( raise litellm.BadRequestError( message=f"Invalid document type: {doc_type}. Must be 'document_url', 'image_url', or 'file'", model=model, - llm_provider=_error_provider(model, custom_llm_provider) or "", + llm_provider=resolve_error_provider(model, custom_llm_provider) or "", ) if not normalized_document.get(doc_type): raise litellm.BadRequestError( message="Document URL is required", model=model, - llm_provider=_error_provider(model, custom_llm_provider) or "", + llm_provider=resolve_error_provider(model, custom_llm_provider) or "", ) ( @@ -182,7 +182,7 @@ def _prepare_ocr_request( ) -def _error_provider(model: str, custom_llm_provider: str | None) -> str | None: +def resolve_error_provider(model: str, custom_llm_provider: str | None) -> str | None: if custom_llm_provider is not None: return custom_llm_provider prefix: Final = model.partition("/")[0] @@ -252,7 +252,7 @@ async def aocr( return response except Exception as e: - error_provider: Final = _error_provider(model, custom_llm_provider) + error_provider: Final = resolve_error_provider(model, custom_llm_provider) error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model raise litellm.exception_type( model=error_model, @@ -412,7 +412,7 @@ def ocr( return response except Exception as e: - error_provider: Final = _error_provider(model, custom_llm_provider) + error_provider: Final = resolve_error_provider(model, custom_llm_provider) error_model: Final = model.removeprefix(f"{error_provider}/") if error_provider else model raise litellm.exception_type( model=error_model, diff --git a/litellm/rust_bridge/failures.py b/litellm/rust_bridge/failures.py index 80805b7ff69..83e33f7ce30 100644 --- a/litellm/rust_bridge/failures.py +++ b/litellm/rust_bridge/failures.py @@ -17,8 +17,8 @@ _UPSTREAM_HEADERS: Final = TypeAdapter(list[tuple[str, str]]) class UpstreamFailure(Exception): def __init__(self, response: httpx.Response, cause: Exception) -> None: - super().__init__(str(cause)) - self.message: Final = str(cause) + super().__init__(response.text) + self.message: Final = response.text self.response: Final = response self.status_code: Final = response.status_code self.__cause__ = cause @@ -49,13 +49,13 @@ class ExceptionMapper(Protocol): ) -> Exception: ... -def map_failure(error: Exception, model: str, request_provider: str, kwargs: Mapping[str, object]) -> Exception: +def map_failure(error: Exception, model: str, request_provider: str | None, kwargs: Mapping[str, object]) -> Exception: mapper: Final = cast( # cast-ok: bounded adapter for the legacy public exception mapper ExceptionMapper, litellm.exception_type ) try: return mapper( - model=model.removeprefix(f"{request_provider}/"), + model=model.removeprefix(f"{request_provider}/") if request_provider else model, custom_llm_provider=request_provider, original_exception=error, completion_kwargs=dict(kwargs), # mutable-ok: exception mapper requires owned kwargs @@ -67,7 +67,11 @@ def map_failure(error: Exception, model: str, request_provider: str, kwargs: Map def map_native_failure( - error: Exception, model: str, request_provider: str, kwargs: Mapping[str, object], api_base: str | None = None + error: Exception, + model: str, + request_provider: str | None, + kwargs: Mapping[str, object], + api_base: str | None = None, ) -> Exception: """`map_failure`, reading a native `(status, body)` provider failure as the HTTP response it was.""" original: Final = _upstream_failure(error, api_base) diff --git a/litellm/rust_bridge/ocr/route_host.py b/litellm/rust_bridge/ocr/route_host.py index bfbd5c11d4e..26b9b5e8fc6 100644 --- a/litellm/rust_bridge/ocr/route_host.py +++ b/litellm/rust_bridge/ocr/route_host.py @@ -8,6 +8,7 @@ from pydantic import TypeAdapter import litellm from litellm.llms.base_llm.ocr.transformation import PROVIDER_NATIVE_RESPONSE_KEY, OCRResponse +from litellm.ocr.main import resolve_error_provider from litellm.rust_bridge import failures from litellm.rust_bridge.failures import UpstreamFailure from litellm.rust_bridge.ocr.entrypoints import LiteLLMOcrRequest @@ -31,11 +32,31 @@ def arguments(request: LiteLLMOcrRequest) -> Mapping[str, object]: return request.kwargs -def map_failure(error: Exception, request: LiteLLMOcrRequest, request_provider: str) -> Exception: +def map_failure(error: Exception, request: LiteLLMOcrRequest, projected_provider: str) -> Exception: + request_provider: Final = projected_provider or resolve_error_provider(request.model, request.custom_llm_provider) if getattr(error, "ocr_request_format_error", False): return litellm.UnsupportedParamsError( - message=f"Invalid `req_format`: {request.kwargs.get('req_format')!r}. Expected 'native' or 'litellm'.", - model=request.model.removeprefix(f"{request_provider}/"), - llm_provider=request_provider, + message=f"Invalid `req_format`: {request.kwargs.get('req_format')!r}. Expected one of litellm, native.", + model=request.model.removeprefix(f"{request_provider}/") if request_provider else request.model, + llm_provider=request_provider or "", ) + invalid_provider: Final = getattr(error, "ocr_invalid_provider", None) + if isinstance(invalid_provider, str): + return _invalid_provider_failure(invalid_provider, request) return failures.map_native_failure(error, request.model, request_provider, arguments(request), request.api_base) + + +def _invalid_provider_failure(provider: str, request: LiteLLMOcrRequest) -> Exception: + """Python rejects a LiteLLM provider without OCR support differently from an unknown provider prefix.""" + if provider in litellm.provider_list: + return failures.map_failure( + ValueError(f"OCR is not supported for provider: {provider}"), request.model, None, arguments(request) + ) + return litellm.BadRequestError( + message=( + "LLM Provider NOT provided. Pass in the LLM provider you are trying to call. " + f"You passed model={request.model}" + ), + model=request.model, + llm_provider="", + ) diff --git a/tests/rust-python-harness/shared/parity/runner.py b/tests/rust-python-harness/shared/parity/runner.py index feae0201aee..453063db9da 100644 --- a/tests/rust-python-harness/shared/parity/runner.py +++ b/tests/rust-python-harness/shared/parity/runner.py @@ -52,13 +52,21 @@ class ExecutionVariant: class SubprocessWorker: - def __init__(self, runner: SubprocessRunner, provider: ReplayServer, variant: ExecutionVariant) -> None: + def __init__( + self, + runner: SubprocessRunner, + provider: ReplayServer, + variant: ExecutionVariant, + *, + baseline: bool, + ) -> None: project_root: Final = str(PROJECT_ROOT) existing_pythonpath: Final = os.environ.get("PYTHONPATH") env: Final = { **os.environ, **dict(variant.environment), - "LITELLM_USER_AGENT": runner.baseline_user_agent, + # Both implementations honor LITELLM_USER_AGENT, so only the baseline carries the sentinel. + **({"LITELLM_USER_AGENT": runner.baseline_user_agent} if baseline else {}), "PYTHONPATH": os.pathsep.join(path for path in (project_root, existing_pythonpath) if path), } self.mode: Final = variant.name @@ -159,9 +167,11 @@ class SubprocessWorker: def execution_worker( runner: SubprocessRunner, variant: ExecutionVariant, + *, + baseline: bool, ) -> Generator[SubprocessWorker]: with replay_server() as provider: - worker: Final = SubprocessWorker(runner, provider, variant) + worker: Final = SubprocessWorker(runner, provider, variant, baseline=baseline) try: yield worker finally: @@ -174,8 +184,8 @@ def execution_worker_pair( baseline: ExecutionVariant, candidate: ExecutionVariant, ) -> Generator[tuple[SubprocessWorker, SubprocessWorker]]: - with execution_worker(runner, baseline) as baseline_worker: - with execution_worker(runner, candidate) as candidate_worker: + with execution_worker(runner, baseline, baseline=True) as baseline_worker: + with execution_worker(runner, candidate, baseline=False) as candidate_worker: yield baseline_worker, candidate_worker diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py index f14296fbf06..0092f2daae6 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/ocr/test_sdk_parity.py @@ -234,14 +234,6 @@ INVALID_OCR_CASES: Final = ( expected_status_code=500, expected_message="OCR is not supported for provider: openai", ), - InvalidOcrCase( - name="unsupported_reducto_model", - model="reducto/parse-v4", - document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, - expected_message="OCR is not supported for provider: reducto", - ), InvalidOcrCase( name="unknown_provider_prefix", model="not_a_provider/model", @@ -254,40 +246,40 @@ INVALID_OCR_CASES: Final = ( name="non_object_document", model="mistral/mistral-ocr-latest", document=[], - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="document must be a dict", ), InvalidOcrCase( name="missing_document_type", model="mistral/mistral-ocr-latest", document={}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Invalid document type: None", ), InvalidOcrCase( name="unsupported_document_type", model="mistral/mistral-ocr-latest", document={"type": "text", "text": "not a document"}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Invalid document type: text", ), InvalidOcrCase( name="missing_document_url", model="azure_ai/doc-intelligence/prebuilt-read", document={"type": "document_url"}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Document URL is required", ), InvalidOcrCase( name="missing_image_url", model="azure_ai/doc-intelligence/prebuilt-read", document={"type": "image_url"}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Document URL is required", ), InvalidOcrCase( @@ -303,8 +295,8 @@ INVALID_OCR_CASES: Final = ( name="invalid_document_intelligence_pages", model="azure_ai/doc-intelligence/prebuilt-read", document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="`pages` integers must be >= 0", extra_kwargs=(("pages", [-1]),), ), @@ -312,8 +304,8 @@ INVALID_OCR_CASES: Final = ( name="invalid_document_intelligence_features", model="azure_ai/doc-intelligence/prebuilt-read", document={"type": "document_url", "document_url": "data:application/pdf;base64,AA=="}, - expected_exception_type="litellm.exceptions.APIConnectionError", - expected_status_code=500, + expected_exception_type="litellm.exceptions.BadRequestError", + expected_status_code=400, expected_message="Invalid `features` for Azure Document Intelligence", extra_kwargs=(("features", [1]),), ), diff --git a/tests/test_litellm/rust_bridge/ocr/test_route_host.py b/tests/test_litellm/rust_bridge/ocr/test_route_host.py index 699492e4424..744fe437ba1 100644 --- a/tests/test_litellm/rust_bridge/ocr/test_route_host.py +++ b/tests/test_litellm/rust_bridge/ocr/test_route_host.py @@ -83,3 +83,33 @@ def test_map_failure_leaves_non_upstream_errors_unwrapped() -> None: def test_map_failure_reports_invalid_request_format_as_unsupported_params() -> None: with pytest.raises(litellm.UnsupportedParamsError, match="Invalid `req_format`: 'markdown'"): raise map_failure(RustFormatError(), REQUEST, "mistral") + + +def test_failure_before_projection_reports_the_provider_python_reports() -> None: + error: Final = ValueError("Document URL is required") + error.status_code = 400 # type: ignore[attr-defined] + + mapped: Final = map_failure(error, REQUEST, "") + + assert isinstance(mapped, litellm.BadRequestError) + assert mapped.llm_provider == "mistral" + assert "BadRequestError: - " not in str(mapped) + + +@pytest.mark.parametrize( + ("provider", "expected_type", "expected_message"), + [ + ("openai", litellm.APIConnectionError, "OCR is not supported for provider: openai"), + ("not_a_provider", litellm.BadRequestError, "LLM Provider NOT provided"), + ], +) +def test_invalid_provider_failures_follow_the_python_route( + provider: str, expected_type: type[Exception], expected_message: str +) -> None: + error: Final = ValueError(f"invalid provider: {provider}") + error.ocr_invalid_provider = provider # type: ignore[attr-defined] + + mapped: Final = map_failure(error, REQUEST, "") + + assert isinstance(mapped, expected_type) + assert expected_message in str(mapped) diff --git a/tests/test_litellm/rust_bridge/test_failures.py b/tests/test_litellm/rust_bridge/test_failures.py index 80057b816d3..5bfe98e699c 100644 --- a/tests/test_litellm/rust_bridge/test_failures.py +++ b/tests/test_litellm/rust_bridge/test_failures.py @@ -52,3 +52,18 @@ def test_kwargs_are_handed_to_the_mapper_as_owned_copies(monkeypatch: pytest.Mon assert seen[0]["completion_kwargs"] is not request_kwargs assert seen[0]["model"] == "gpt-4o" assert seen[0]["custom_llm_provider"] == "openai" + + +def test_native_provider_failure_message_is_the_provider_body_not_the_args_tuple() -> None: + class NativeUpstream(Exception): + headers = [("retry-after", "7")] + + native_error: Final = NativeUpstream(429, '{"message": "slow down"}') + + mapped: Final = failures.map_native_failure( + native_error, "mistral/mistral-ocr-latest", "mistral", MappingProxyType({}), "https://api.mistral.ai/v1" + ) + + assert isinstance(mapped, litellm.RateLimitError) + assert '{"message": "slow down"}' in str(mapped) + assert "(429," not in str(mapped) diff --git a/tests/test_litellm/rust_bridge/test_loader.py b/tests/test_litellm/rust_bridge/test_loader.py new file mode 100644 index 00000000000..a5bd56d0a6a --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_loader.py @@ -0,0 +1,27 @@ +from types import ModuleType + +import pytest + +from litellm.rust_bridge import loader + + +@pytest.fixture(autouse=True) +def _reset_loader() -> None: + loader.reset_native_bridge_cache() + yield + loader.reset_native_bridge_cache() + + +def test_child_forked_after_the_extension_loaded_reports_it_unavailable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(loader, "_cached_bridge", ModuleType("_native")) + + loader._disable_in_forked_child() + + assert loader.get_native_bridge() is None + assert loader.native_bridge_available() is False + + +def test_child_forked_before_the_extension_loaded_keeps_it() -> None: + loader._disable_in_forked_child() + + assert loader._forked_after_load is False