From 657cf18aea50d8825b629d95f9e74467d17391d9 Mon Sep 17 00:00:00 2001 From: Yujong Lee Date: Fri, 18 Sep 2026 13:01:55 -0700 Subject: [PATCH] feat(rust): port exception_type to litellm-core-utils Adds a Rust port of litellm_core_utils/exception_mapping_utils.exception_type with its provider rule tables (OpenAI-compatible, Cohere, Vertex AI), the secret redaction patterns from secret_redaction.py, and a Python repr helper for messages that quote caller values. The public failure shapes are pinned by golden JSON fixtures under tests/test_litellm/rust_bridge/fixtures/public_failures, which the Python side reads too once the bridge is wired to this module. Nothing calls the port yet. --- litellm-rust/Cargo.lock | 29 + litellm-rust/Cargo.toml | 1 + litellm-rust/crates/core-utils/Cargo.toml | 5 + .../src/exception_mapping_utils/cohere.rs | 232 ++++++ .../src/exception_mapping_utils/mod.rs | 734 ++++++++++++++++++ .../src/exception_mapping_utils/openai.rs | 586 ++++++++++++++ .../src/exception_mapping_utils/original.rs | 49 ++ .../src/exception_mapping_utils/public.rs | 262 +++++++ .../src/exception_mapping_utils/rules.rs | 403 ++++++++++ .../src/exception_mapping_utils/status.rs | 153 ++++ .../src/exception_mapping_utils/vertex_ai.rs | 574 ++++++++++++++ litellm-rust/crates/core-utils/src/lib.rs | 3 + .../crates/core-utils/src/python_repr.rs | 93 +++ .../crates/core-utils/src/secret_redaction.rs | 97 +++ .../fixtures/public_failures/api.json | 13 + .../public_failures/api_connection.json | 11 + .../status_authentication.json | 13 + .../public_failures/status_bad_gateway.json | 28 + .../public_failures/status_bad_request.json | 28 + .../status_content_policy_violation.json | 28 + .../status_context_window_exceeded.json | 28 + .../status_internal_server.json | 19 + .../public_failures/status_not_found.json | 28 + .../status_permission_denied.json | 19 + .../public_failures/status_rate_limit.json | 28 + .../status_service_unavailable.json | 28 + .../status_unsupported_params.json | 28 + .../public_failures/timeout_with_status.json | 12 + .../timeout_without_status.json | 12 + 29 files changed, 3544 insertions(+) create mode 100644 litellm-rust/crates/core-utils/src/exception_mapping_utils/cohere.rs create mode 100644 litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs create mode 100644 litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs create mode 100644 litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs create mode 100644 litellm-rust/crates/core-utils/src/exception_mapping_utils/public.rs create mode 100644 litellm-rust/crates/core-utils/src/exception_mapping_utils/rules.rs create mode 100644 litellm-rust/crates/core-utils/src/exception_mapping_utils/status.rs create mode 100644 litellm-rust/crates/core-utils/src/exception_mapping_utils/vertex_ai.rs create mode 100644 litellm-rust/crates/core-utils/src/python_repr.rs create mode 100644 litellm-rust/crates/core-utils/src/secret_redaction.rs create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/api.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/api_connection.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_authentication.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_bad_gateway.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_bad_request.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_content_policy_violation.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_context_window_exceeded.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_internal_server.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_not_found.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_permission_denied.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_rate_limit.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_service_unavailable.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/status_unsupported_params.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/timeout_with_status.json create mode 100644 tests/test_litellm/rust_bridge/fixtures/public_failures/timeout_without_status.json diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 3b9ff62fbaa..6c524124550 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -559,6 +559,21 @@ dependencies = [ "vsimd", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bitflags" version = "2.13.1" @@ -1166,6 +1181,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "fancy-regex" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d301f5bf187b3c295fce6468d3875037a0bccc5f6b151c63cac2f85babf21912" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -2059,11 +2085,14 @@ dependencies = [ name = "litellm-core-utils" version = "0.1.0" dependencies = [ + "fancy-regex", "litellm-types", + "rstest", "serde", "serde_json", "serde_path_to_error", "serde_with", + "strum", "thiserror 2.0.19", "url", ] diff --git a/litellm-rust/Cargo.toml b/litellm-rust/Cargo.toml index f8377138050..ffdbf64bb49 100644 --- a/litellm-rust/Cargo.toml +++ b/litellm-rust/Cargo.toml @@ -50,6 +50,7 @@ strum = { version = "0.28.0", features = ["derive"] } url = "2.5.8" time = { version = "0.3.53", features = ["parsing"] } criterion = "0.8.2" +fancy-regex = "0.19.2" veil = "0.3.0" [profile.release] diff --git a/litellm-rust/crates/core-utils/Cargo.toml b/litellm-rust/crates/core-utils/Cargo.toml index 109c3312727..59e0ee1a09d 100644 --- a/litellm-rust/crates/core-utils/Cargo.toml +++ b/litellm-rust/crates/core-utils/Cargo.toml @@ -6,10 +6,15 @@ license.workspace = true repository.workspace = true [dependencies] +fancy-regex.workspace = true litellm-types.workspace = true serde.workspace = true serde_json.workspace = true serde_path_to_error = "0.1" serde_with.workspace = true +strum.workspace = true thiserror.workspace = true url.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/cohere.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/cohere.rs new file mode 100644 index 00000000000..381ebd7ad27 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/cohere.rs @@ -0,0 +1,232 @@ +use super::Mapping; +use super::public::{PublicFailure, StatusClass}; +use super::rules::{Kind, ResponseChoice, Rule, apply, contains_any}; + +const fn with_response(class: StatusClass) -> Kind { + Kind::Status { + class, + response: ResponseChoice::Provider, + } +} + +fn original(mapping: &Mapping<'_>) -> String { + format!("CohereException - {}", mapping.original.message) +} + +fn status_is(mapping: &Mapping<'_>, statuses: &[u16]) -> bool { + mapping + .original + .status + .is_some_and(|status| statuses.contains(&status)) +} + +/// `_map_cohere_exception`, in its branch order. A failure no rule claims falls through to +/// the status table. +const RULES: &[Rule] = &[ + Rule { + when: |mapping| { + contains_any( + &mapping.error_str, + &["invalid api token", "No API key provided."], + ) + }, + kind: with_response(StatusClass::Authentication), + message: original, + debug: false, + }, + Rule { + when: |mapping| mapping.error_str.contains("invalid type: parameter"), + kind: with_response(StatusClass::BadRequest), + message: original, + debug: false, + }, + Rule { + when: |mapping| mapping.error_str.contains("too many tokens"), + kind: with_response(StatusClass::ContextWindowExceeded), + message: original, + debug: false, + }, + Rule { + when: |mapping| { + mapping + .error_str + .to_lowercase() + .contains("internal server error") + }, + kind: with_response(StatusClass::InternalServer), + message: |mapping| format!("CohereException - {}", mapping.error_str), + debug: false, + }, + Rule { + when: |mapping| status_is(mapping, &[400, 498]), + kind: with_response(StatusClass::BadRequest), + message: original, + debug: false, + }, + Rule { + when: |mapping| status_is(mapping, &[408]), + kind: Kind::Timeout(None), + message: original, + debug: false, + }, + Rule { + when: |mapping| status_is(mapping, &[500]), + kind: with_response(StatusClass::InternalServer), + message: original, + debug: false, + }, +]; + +pub(super) fn map(mapping: &Mapping<'_>) -> Option { + apply(RULES, mapping).map(|failure| PublicFailure { + llm_provider: Some("cohere".to_string()), + ..failure + }) +} + +#[cfg(test)] +mod tests { + use super::super::testing::{context, failure, http, status, upstream}; + use super::super::{ExceptionFamily, OriginalException, PublicKind}; + use super::*; + + fn mapped(provider: &str, original: &OriginalException) -> Option { + let context = context(provider, ExceptionFamily::Cohere); + map(&Mapping::new(&context, original)) + } + + fn cohere(class: StatusClass, status_code: u16, body: &str, message: &str) -> PublicFailure { + failure( + status(class, upstream(status_code, body)), + message, + "cohere", + ) + } + + #[rstest::rstest] + #[case::invalid_token( + 500, + "invalid api token", + cohere( + StatusClass::Authentication, + 500, + "invalid api token", + "CohereException - invalid api token" + ) + )] + #[case::no_api_key( + 500, + "No API key provided.", + cohere( + StatusClass::Authentication, + 500, + "No API key provided.", + "CohereException - No API key provided." + ) + )] + #[case::invalid_parameter( + 500, + "invalid type: parameter x", + cohere( + StatusClass::BadRequest, + 500, + "invalid type: parameter x", + "CohereException - invalid type: parameter x" + ) + )] + #[case::too_many_tokens( + 500, + "too many tokens", + cohere( + StatusClass::ContextWindowExceeded, + 500, + "too many tokens", + "CohereException - too many tokens" + ) + )] + #[case::internal_server_text( + 400, + "Internal Server Error", + cohere( + StatusClass::InternalServer, + 400, + "Internal Server Error", + "CohereException - Internal Server Error" + ) + )] + #[case::bad_request( + 400, + "rejected", + cohere(StatusClass::BadRequest, 400, "rejected", "CohereException - rejected") + )] + #[case::invalid_token_status( + 498, + "rejected", + cohere(StatusClass::BadRequest, 498, "rejected", "CohereException - rejected") + )] + #[case::request_timeout(408, "rejected", failure(PublicKind::Timeout { status: None }, "CohereException - rejected", "cohere"))] + #[case::internal_server( + 500, + "rejected", + cohere( + StatusClass::InternalServer, + 500, + "rejected", + "CohereException - rejected" + ) + )] + fn each_rule_maps_and_reports_cohere( + #[case] status_code: u16, + #[case] body: &str, + #[case] expected: PublicFailure, + ) { + assert_eq!(mapped("azure_ai", &http(status_code, body)), Some(expected)); + } + + #[rstest::rstest] + #[case::unmapped_status(409)] + #[case::unauthorized(401)] + fn statuses_without_a_rule_fall_through(#[case] status_code: u16) { + assert_eq!(mapped("cohere", &http(status_code, "rejected")), None); + } + + #[test] + fn the_internal_server_rule_uses_the_redacted_text() { + let body = "internal server error Bearer abcdefghijklmnop"; + assert_eq!( + mapped("cohere", &http(400, body)), + Some(cohere( + StatusClass::InternalServer, + 400, + body, + "CohereException - internal server error REDACTED" + )) + ); + } + + #[rstest::rstest] + #[case::token_before_parameter( + "invalid api token invalid type: parameter", + StatusClass::Authentication + )] + #[case::parameter_before_tokens( + "invalid type: parameter too many tokens", + StatusClass::BadRequest + )] + #[case::tokens_before_internal( + "too many tokens Internal Server Error", + StatusClass::ContextWindowExceeded + )] + #[case::internal_before_status("Internal Server Error", StatusClass::InternalServer)] + fn the_earlier_rule_wins_when_two_apply(#[case] body: &str, #[case] class: StatusClass) { + assert_eq!( + mapped("cohere", &http(400, body)), + Some(cohere( + class, + 400, + body, + &format!("CohereException - {body}") + )) + ); + } +} diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs new file mode 100644 index 00000000000..8892fba1399 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/mod.rs @@ -0,0 +1,734 @@ +//! A port of Python's `exception_type` for the routes that run in Rust. +//! +//! KNOWN_GAPS: differences from the Python mapper that no Rust route can reach today. Each +//! one stops being acceptable at its trigger. +//! - The Vertex partner-model API base for "claude" models is not built into +//! `extra_information`. Trigger: a Vertex route whose models include Anthropic partner +//! models; then `api_base` gets that branch and a table row. +//! - Python reports the provider `get_llm_provider` resolves for a stripped model name when +//! that name happens to be in the model cost map. Trigger: a route whose model names +//! overlap the cost map; that needs the provider resolution port, not a classifier change. +//! - The generic `APIConnectionError` fallback appends `traceback.format_exc()` to the +//! message. Rust has no Python traceback and does not invent one; a sweep row that reaches +//! it compares the message before the traceback. + +use super::secret_redaction::{redact_string, secret_redaction_enabled}; + +mod cohere; +mod openai; +mod original; +mod public; +mod rules; +mod status; +mod vertex_ai; + +pub use original::{ExceptionFamily, LocalClass, OriginalException}; +pub use public::{HttpStub, PublicFailure, PublicKind, ResponseArg, StatusClass, UpstreamResponse}; + +const DOCS_URL: &str = "https://docs.litellm.ai/docs"; + +const TIMEOUT_MARKERS: &[&str] = &[ + "Request Timeout Error", + "Request timed out", + "Timed out generating response", + "The read operation timed out", +]; + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ExceptionContext { + pub model: String, + pub custom_llm_provider: Option, + pub family: ExceptionFamily, + pub asynchronous: bool, + pub suppress_debug_info: bool, + pub redact_messages_in_exceptions: bool, + pub vertex_project: Option, + pub vertex_location: Option, + pub model_group: Option, + pub deployment: Option, + pub user_api_key_alias: Option, + pub user_api_key_team_alias: Option, +} + +/// The attributes `exception_type` reads off the Python exception: a provider error +/// (`BaseLLMException`) carries a status, a response and a request, a plain exception +/// carries only its text. +struct Raised { + status: Option, + status_is_synthesized: bool, + message: String, + response: Option, +} + +impl Raised { + fn provider( + status: u16, + message: String, + body: String, + headers: Vec<(String, String)>, + ) -> Self { + Self { + status: Some(status), + status_is_synthesized: false, + message, + response: Some(UpstreamResponse { + status, + body, + headers, + }), + } + } + + fn plain(message: String) -> Self { + Self { + status: None, + status_is_synthesized: false, + message, + response: None, + } + } + + fn new(original: &OriginalException, asynchronous: bool) -> Self { + match original { + OriginalException::Http { + status, + body, + headers, + } => Self::provider(*status, body.clone(), body.clone(), headers.clone()), + OriginalException::Connection { message } => Self { + status_is_synthesized: true, + ..Self::provider(500, message.clone(), String::new(), Vec::new()) + }, + OriginalException::Timeout { + timeout_seconds, + elapsed_seconds, + } => Self::provider( + 408, + timeout_message(asynchronous, *timeout_seconds, *elapsed_seconds), + String::new(), + Vec::new(), + ), + OriginalException::Response { message } + | OriginalException::Local { message, .. } + | OriginalException::Public { message, .. } => Self::plain(message.clone()), + } + } +} + +/// The text `litellm.Timeout` carries when the Python HTTP handler times out: the sync +/// and async handlers word it differently. +fn timeout_message( + asynchronous: bool, + timeout_seconds: Option, + elapsed_seconds: Option, +) -> String { + let timeout = python_float(timeout_seconds); + if asynchronous { + let elapsed = + python_float(elapsed_seconds.map(|seconds| (seconds * 1000.0).round() / 1000.0)); + format!( + "litellm.Timeout: Connection timed out. Timeout passed={timeout}, time taken={elapsed} seconds" + ) + } else { + format!("litellm.Timeout: Connection timed out after {timeout} seconds.") + } +} + +fn python_float(value: Option) -> String { + match value { + None => "None".to_string(), + Some(value) if value.fract() == 0.0 => format!("{value:.1}"), + Some(value) => value.to_string(), + } +} + +/// Everything the rules read: the original as Python sees it and the text `exception_type` +/// derives from the context before any provider mapper runs. +struct Mapping<'a> { + context: &'a ExceptionContext, + original: Raised, + provider: &'a str, + error_str: String, + exception_provider: String, + extra_information: String, +} + +impl<'a> Mapping<'a> { + fn new(context: &'a ExceptionContext, original: &OriginalException) -> Self { + let original = Raised::new(original, context.asynchronous); + let error_str = if secret_redaction_enabled() { + redact_string(&original.message) + } else { + original.message.clone() + }; + Self { + context, + original, + provider: context.custom_llm_provider.as_deref().unwrap_or_default(), + error_str, + exception_provider: match &context.custom_llm_provider { + None => "None".to_string(), + Some(provider) => exception_provider(provider), + }, + extra_information: extra_information(context, api_base(context).as_deref()), + } + } + + fn failure(&self, kind: PublicKind, message: String, debug: bool) -> PublicFailure { + PublicFailure { + kind, + message, + model: self.context.model.clone(), + llm_provider: self.context.custom_llm_provider.clone(), + litellm_debug_info: debug.then(|| self.extra_information.clone()), + litellm_response_headers: None, + print_banner: false, + } + } +} + +pub fn exception_type(context: &ExceptionContext, original: &OriginalException) -> PublicFailure { + if let OriginalException::Public { class, message } = original { + return PublicFailure { + kind: PublicKind::Status { + status_class: *class, + response: None, + }, + message: message.clone(), + model: context.model.clone(), + llm_provider: context.custom_llm_provider.clone(), + litellm_debug_info: None, + litellm_response_headers: None, + print_banner: false, + }; + } + let mapping = Mapping::new(context, original); + let litellm_response_headers = mapping + .original + .response + .as_ref() + .map(|response| response.headers.clone()) + .filter(|headers| !headers.is_empty()); + PublicFailure { + litellm_response_headers, + print_banner: !context.suppress_debug_info, + ..map(&mapping) + } +} + +fn map(mapping: &Mapping<'_>) -> PublicFailure { + if rules::contains_any(&mapping.error_str, TIMEOUT_MARKERS) { + return mapping.failure( + PublicKind::Timeout { status: None }, + format!( + "APITimeoutError - Request timed out. Error_str: {}", + mapping.error_str + ), + true, + ); + } + let provider_failure = match mapping.context.family { + ExceptionFamily::OpenAiCompatible => openai::map(mapping), + ExceptionFamily::VertexAi => vertex_ai::map(mapping), + ExceptionFamily::Cohere => cohere::map(mapping), + ExceptionFamily::Other => None, + }; + provider_failure + .or_else(|| status::map(mapping)) + .unwrap_or_else(|| unmapped(mapping)) +} + +/// The `APIConnectionError` Python raises when no mapper claimed the failure: with the +/// provider prefix for a provider error, with the bare text for a plain exception. +fn unmapped(mapping: &Mapping<'_>) -> PublicFailure { + let message = match mapping.original.status { + Some(_) => format!("{} - {}", mapping.exception_provider, mapping.error_str), + None => mapping.original.message.clone(), + }; + mapping.failure(PublicKind::ApiConnection, message, false) +} + +fn exception_provider(provider: &str) -> String { + let mut characters = provider.chars(); + match characters.next() { + Some(first) => format!("{}{}Exception", first.to_uppercase(), characters.as_str()), + None => String::new(), + } +} + +fn python_capitalize(value: &str) -> String { + let mut characters = value.chars(); + match characters.next() { + Some(first) => format!( + "{}{}", + first.to_uppercase(), + characters.as_str().to_lowercase() + ), + None => String::new(), + } +} + +fn api_base(context: &ExceptionContext) -> Option { + match (&context.vertex_location, &context.vertex_project) { + (Some(location), Some(project)) => Some(format!( + "{location}-aiplatform.googleapis.com/v1/projects/{project}/locations/{location}/publishers/google/models/{}:generateContent", + context.model + )), + _ => None, + } +} + +fn extra_information(context: &ExceptionContext, api_base: Option<&str>) -> String { + let lines = [ + Some(format!("\nModel: {}", context.model)), + api_base.map(|api_base| format!("\nAPI Base: `{api_base}`")), + (!context.redact_messages_in_exceptions).then(|| "\nMessages: `None`".to_string()), + context + .model_group + .as_ref() + .map(|value| format!("\nmodel_group: `{value}`\n")), + context + .deployment + .as_ref() + .map(|value| format!("\ndeployment: `{value}`\n")), + context + .vertex_project + .as_ref() + .map(|value| format!("\nvertex_project: `{value}`\n")), + context + .vertex_location + .as_ref() + .map(|value| format!("\nvertex_location: `{value}`\n")), + ]; + let information: String = lines.into_iter().flatten().collect(); + match &context.user_api_key_alias { + Some(alias) => format!( + "\n\nKey Name: `{alias}`\nTeam: `{}`{information}", + context.user_api_key_team_alias.as_deref().unwrap_or("None") + ), + None => information, + } +} + +#[cfg(test)] +mod testing { + use super::*; + + pub(super) const DEBUG: &str = "\nModel: ocr-model\nMessages: `None`"; + + pub(super) fn context(provider: &str, family: ExceptionFamily) -> ExceptionContext { + ExceptionContext { + model: "ocr-model".into(), + custom_llm_provider: Some(provider.into()), + family, + suppress_debug_info: true, + ..ExceptionContext::default() + } + } + + pub(super) fn http(status: u16, body: &str) -> OriginalException { + OriginalException::Http { + status, + body: body.into(), + headers: vec![("retry-after".into(), "7".into())], + } + } + + pub(super) fn upstream(status: u16, body: &str) -> Option { + Some(ResponseArg::Upstream(UpstreamResponse { + status, + body: body.into(), + headers: vec![("retry-after".into(), "7".into())], + })) + } + + pub(super) fn status(class: StatusClass, response: Option) -> PublicKind { + PublicKind::Status { + status_class: class, + response, + } + } + + /// The failure a rule builds before `exception_type` adds the response headers and the + /// banner flag. + pub(super) fn failure(kind: PublicKind, message: &str, provider: &str) -> PublicFailure { + PublicFailure { + kind, + message: message.into(), + model: "ocr-model".into(), + llm_provider: Some(provider.into()), + litellm_debug_info: None, + litellm_response_headers: None, + print_banner: false, + } + } + + pub(super) fn with_debug(failure: PublicFailure) -> PublicFailure { + PublicFailure { + litellm_debug_info: Some(DEBUG.into()), + ..failure + } + } + + /// What `exception_type` returns for an `http` original the rule mapped to `failure`. + pub(super) fn with_headers(failure: PublicFailure) -> PublicFailure { + PublicFailure { + litellm_response_headers: Some(vec![("retry-after".into(), "7".into())]), + ..failure + } + } +} + +#[cfg(test)] +mod tests { + use super::testing::{DEBUG, context, failure, http, status, upstream, with_debug}; + use super::*; + + fn openai() -> ExceptionContext { + context("mistral", ExceptionFamily::OpenAiCompatible) + } + + #[test] + fn a_public_original_passes_through_without_banner_debug_or_prefix() { + let original = OriginalException::Public { + class: StatusClass::UnsupportedParams, + message: "Invalid `req_format`".into(), + }; + let context = ExceptionContext { + suppress_debug_info: false, + ..openai() + }; + assert_eq!( + exception_type(&context, &original), + failure( + status(StatusClass::UnsupportedParams, None), + "Invalid `req_format`", + "mistral" + ) + ); + } + + #[rstest::rstest] + #[case::vertex_family_status_rule(ExceptionFamily::VertexAi, "vertex_ai", PublicFailure { + litellm_response_headers: Some(vec![("retry-after".into(), "7".into())]), + ..with_debug(failure(status(StatusClass::BadRequest, upstream(409, "rejected")), "Vertex_aiException - rejected", "vertex_ai")) + })] + #[case::cohere_family(ExceptionFamily::Cohere, "cohere", PublicFailure { + litellm_response_headers: Some(vec![("retry-after".into(), "7".into())]), + ..with_debug(failure(status(StatusClass::BadRequest, upstream(409, "rejected")), "CohereException - rejected", "cohere")) + })] + #[case::other_family(ExceptionFamily::Other, "reducto", PublicFailure { + litellm_response_headers: Some(vec![("retry-after".into(), "7".into())]), + ..with_debug(failure(status(StatusClass::BadRequest, upstream(409, "rejected")), "ReductoException - rejected", "reducto")) + })] + fn families_without_a_409_rule_reach_the_status_table( + #[case] family: ExceptionFamily, + #[case] provider: &str, + #[case] expected: PublicFailure, + ) { + assert_eq!( + exception_type(&context(provider, family), &http(409, "rejected")), + expected + ); + } + + #[test] + fn the_openai_family_claims_a_409_before_the_status_table() { + assert_eq!( + exception_type(&openai(), &http(409, "rejected")), + PublicFailure { + litellm_response_headers: Some(vec![("retry-after".into(), "7".into())]), + ..with_debug(failure( + PublicKind::Api { + status: 409, + request_url: DOCS_URL + }, + "APIError: MistralException - rejected", + "mistral" + )) + } + ); + } + + #[rstest::rstest] + #[case::request_timeout_error("Request Timeout Error")] + #[case::request_timed_out("Request timed out")] + #[case::timed_out_generating("Timed out generating response")] + #[case::read_operation("The read operation timed out")] + fn timeout_markers_win_over_every_family(#[case] marker: &str) { + let body = format!("rate limit {marker}"); + for family in [ + ExceptionFamily::OpenAiCompatible, + ExceptionFamily::VertexAi, + ExceptionFamily::Cohere, + ExceptionFamily::Other, + ] { + assert_eq!( + exception_type(&context("mistral", family), &http(429, &body)), + PublicFailure { + litellm_response_headers: Some(vec![("retry-after".into(), "7".into())]), + ..with_debug(failure( + PublicKind::Timeout { status: None }, + &format!("APITimeoutError - Request timed out. Error_str: {body}"), + "mistral" + )) + } + ); + } + } + + #[rstest::rstest] + #[case::provider_error_keeps_the_prefix(http(409, "rejected"), "ReductoException - rejected")] + #[case::synthesized_status_skips_the_status_table( + OriginalException::Connection { message: "refused".into() }, + "ReductoException - refused" + )] + #[case::plain_exception_keeps_its_text( + OriginalException::Local { class: LocalClass::FileNotFound, message: "File not found: /a".into() }, + "File not found: /a" + )] + fn unmapped_failures_are_connection_errors( + #[case] original: OriginalException, + #[case] message: &str, + ) { + let context = context("reducto", ExceptionFamily::Other); + let expected = failure(PublicKind::ApiConnection, message, "reducto"); + let actual = exception_type(&context, &original); + assert_eq!( + PublicFailure { + litellm_response_headers: None, + ..actual + }, + expected + ); + } + + #[test] + fn a_missing_provider_renders_like_python_none() { + let context = ExceptionContext { + custom_llm_provider: None, + family: ExceptionFamily::Other, + ..openai() + }; + assert_eq!( + exception_type(&context, &http(401, "rejected")), + PublicFailure { + llm_provider: None, + litellm_response_headers: Some(vec![("retry-after".into(), "7".into())]), + ..with_debug(failure( + status(StatusClass::Authentication, upstream(401, "rejected")), + "None - rejected", + "unused" + )) + } + ); + } + + #[rstest::rstest] + #[case::suppressed(true, false)] + #[case::printed(false, true)] + fn the_banner_prints_unless_debug_info_is_suppressed( + #[case] suppress_debug_info: bool, + #[case] print_banner: bool, + ) { + let context = ExceptionContext { + suppress_debug_info, + ..openai() + }; + assert_eq!( + exception_type(&context, &http(400, "rejected")).print_banner, + print_banner + ); + } + + #[test] + fn empty_upstream_headers_are_not_reported() { + let original = OriginalException::Http { + status: 400, + body: "rejected".into(), + headers: Vec::new(), + }; + assert_eq!( + exception_type(&openai(), &original).litellm_response_headers, + None + ); + } + + #[test] + fn messages_are_redacted_before_markers_and_prefixes() { + let body = "rejected Bearer abcdefghijklmnop"; + assert_eq!( + exception_type( + &context("reducto", ExceptionFamily::Other), + &http(400, body) + ) + .message, + "ReductoException - rejected REDACTED" + ); + } + + const SYNC_TIMEOUT: &str = "litellm.Timeout: Connection timed out after 0.5 seconds."; + + #[rstest::rstest] + #[case::sync(false, Some(0.5), Some(0.5031), SYNC_TIMEOUT)] + #[case::async_rounds_the_elapsed_time( + true, + Some(0.5), + Some(0.5031), + "litellm.Timeout: Connection timed out. Timeout passed=0.5, time taken=0.503 seconds" + )] + #[case::whole_seconds_keep_a_decimal( + true, + Some(600.0), + Some(2.0), + "litellm.Timeout: Connection timed out. Timeout passed=600.0, time taken=2.0 seconds" + )] + #[case::unknown_values_render_as_none( + true, + None, + None, + "litellm.Timeout: Connection timed out. Timeout passed=None, time taken=None seconds" + )] + fn timeout_text_follows_the_delivery_mode( + #[case] asynchronous: bool, + #[case] timeout_seconds: Option, + #[case] elapsed_seconds: Option, + #[case] expected: &str, + ) { + assert_eq!( + timeout_message(asynchronous, timeout_seconds, elapsed_seconds), + expected + ); + } + + #[rstest::rstest] + #[case::sync(false, SYNC_TIMEOUT)] + #[case::async_( + true, + "litellm.Timeout: Connection timed out. Timeout passed=0.5, time taken=0.503 seconds" + )] + fn a_timeout_is_a_408_carrying_the_handler_text( + #[case] asynchronous: bool, + #[case] text: &str, + ) { + let context = ExceptionContext { + asynchronous, + ..openai() + }; + let original = OriginalException::Timeout { + timeout_seconds: Some(0.5), + elapsed_seconds: Some(0.5031), + }; + assert_eq!( + exception_type(&context, &original), + with_debug(failure( + PublicKind::Timeout { status: None }, + &format!("Timeout Error: MistralException - {text}"), + "mistral" + )) + ); + } + + #[test] + fn a_refused_connection_is_a_500_with_an_empty_response() { + assert_eq!( + exception_type( + &openai(), + &OriginalException::Connection { + message: "refused".into() + } + ), + with_debug(failure( + status( + StatusClass::InternalServer, + Some(ResponseArg::Upstream(UpstreamResponse { + status: 500, + body: String::new(), + headers: Vec::new(), + })) + ), + "InternalServerError: MistralException - refused", + "mistral" + )) + ); + } + + #[test] + fn debug_information_follows_the_python_layout() { + let context = ExceptionContext { + vertex_project: Some("project".into()), + vertex_location: Some("region".into()), + model_group: Some("ocr".into()), + deployment: Some("deployment".into()), + user_api_key_alias: Some("key".into()), + ..openai() + }; + assert_eq!( + extra_information(&context, api_base(&context).as_deref()), + concat!( + "\n\nKey Name: `key`\nTeam: `None`", + "\nModel: ocr-model", + "\nAPI Base: `region-aiplatform.googleapis.com/v1/projects/project/locations/region/publishers/google/models/ocr-model:generateContent`", + "\nMessages: `None`", + "\nmodel_group: `ocr`\n", + "\ndeployment: `deployment`\n", + "\nvertex_project: `project`\n", + "\nvertex_location: `region`\n", + ) + ); + } + + #[rstest::rstest] + #[case::bare(ExceptionContext::default(), "\nModel: ")] + #[case::redacted_messages(ExceptionContext { redact_messages_in_exceptions: true, model: "m".into(), ..ExceptionContext::default() }, "\nModel: m")] + #[case::messages(ExceptionContext { model: "m".into(), ..ExceptionContext::default() }, "\nModel: m\nMessages: `None`")] + #[case::team_alias( + ExceptionContext { model: "m".into(), user_api_key_alias: Some("key".into()), user_api_key_team_alias: Some("team".into()), redact_messages_in_exceptions: true, ..ExceptionContext::default() }, + "\n\nKey Name: `key`\nTeam: `team`\nModel: m" + )] + #[case::team_alias_without_key_is_ignored( + ExceptionContext { model: "m".into(), user_api_key_team_alias: Some("team".into()), redact_messages_in_exceptions: true, ..ExceptionContext::default() }, + "\nModel: m" + )] + #[case::project_without_location_has_no_api_base( + ExceptionContext { model: "m".into(), vertex_project: Some("p".into()), redact_messages_in_exceptions: true, ..ExceptionContext::default() }, + "\nModel: m\nvertex_project: `p`\n" + )] + #[case::location_without_project_has_no_api_base( + ExceptionContext { model: "m".into(), vertex_location: Some("l".into()), redact_messages_in_exceptions: true, ..ExceptionContext::default() }, + "\nModel: m\nvertex_location: `l`\n" + )] + fn each_optional_context_field_adds_its_own_line( + #[case] context: ExceptionContext, + #[case] expected: &str, + ) { + assert_eq!( + extra_information(&context, api_base(&context).as_deref()), + expected + ); + } + + #[rstest::rstest] + #[case::lowercase("mistral", "MistralException")] + #[case::keeps_the_rest("azure_ai", "Azure_aiException")] + #[case::empty("", "")] + fn exception_provider_capitalizes_only_the_first_letter( + #[case] provider: &str, + #[case] expected: &str, + ) { + assert_eq!(exception_provider(provider), expected); + } + + #[rstest::rstest] + #[case::lowers_the_rest("vERTEX_AI", "Vertex_ai")] + #[case::empty("", "")] + fn python_capitalize_lowers_the_rest(#[case] value: &str, #[case] expected: &str) { + assert_eq!(python_capitalize(value), expected); + } + + #[test] + fn debug_constant_matches_the_default_test_context() { + let context = openai(); + assert_eq!(extra_information(&context, None), DEBUG); + } +} diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs new file mode 100644 index 00000000000..ffbb172582b --- /dev/null +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/openai.rs @@ -0,0 +1,586 @@ +use super::public::{PublicFailure, StatusClass}; +use super::rules::{ + ApiStatus, Kind, ResponseChoice, Rule, apply, contains_any, is_context_window_exceeded, + is_rate_limit, +}; +use super::{DOCS_URL, Mapping}; + +const OPENAI_URL: &str = "https://api.openai.com/v1"; + +const ENCRYPTED_CONTENT_HELP: &str = "\n\n This error occurs when load balancing Responses API across deployments with different API keys.\n Encrypted content is tied to the organization that created it and cannot be decrypted by other organizations.\n\n Solution: Enable 'encrypted_content_affinity' to route follow-up requests to the correct deployment:\n\n router_settings:\n enable_pre_call_checks: true\n optional_pre_call_checks:\n - encrypted_content_affinity\n\n Learn more: https://docs.litellm.ai/docs/response_api#encrypted-content-affinity-multi-region-load-balancing"; + +const fn with_response(class: StatusClass) -> Kind { + Kind::Status { + class, + response: ResponseChoice::Provider, + } +} + +fn exception_provider(mapping: &Mapping<'_>) -> String { + if mapping.provider == "openai" { + "OpenAIException".to_string() + } else { + super::exception_provider(mapping.provider) + } +} + +/// The raw message with OpenAI's own names swapped for the provider's. +fn message(mapping: &Mapping<'_>) -> String { + let provider = mapping.provider; + mapping + .original + .message + .replace("OPENAI", &provider.to_uppercase()) + .replace("openai.OpenAIError", &format!("{provider}.{provider}Error")) +} + +fn prefixed(mapping: &Mapping<'_>, label: &str) -> String { + format!( + "{label}{} - {}", + exception_provider(mapping), + message(mapping) + ) +} + +fn status_is(mapping: &Mapping<'_>, statuses: &[u16]) -> bool { + mapping + .original + .status + .is_some_and(|status| statuses.contains(&status)) +} + +/// `_map_openai_exception`, in its branch order. +const RULES: &[Rule] = &[ + Rule { + when: |mapping| is_rate_limit(&mapping.error_str, mapping.original.status), + kind: with_response(StatusClass::RateLimit), + message: |mapping| prefixed(mapping, "RateLimitError: "), + debug: false, + }, + Rule { + when: |mapping| is_context_window_exceeded(&mapping.error_str), + kind: with_response(StatusClass::ContextWindowExceeded), + message: |mapping| prefixed(mapping, "ContextWindowExceededError: "), + debug: true, + }, + Rule { + when: |mapping| { + mapping.error_str.contains("invalid_request_error") + && mapping.error_str.contains("model_not_found") + }, + kind: with_response(StatusClass::NotFound), + message: |mapping| prefixed(mapping, ""), + debug: true, + }, + Rule { + when: |mapping| mapping.error_str.contains("A timeout occurred"), + kind: Kind::Timeout(None), + message: |mapping| prefixed(mapping, ""), + debug: true, + }, + Rule { + when: |mapping| { + let error_str = &mapping.error_str; + (error_str.contains("invalid_request_error") + && error_str.contains("content_policy_violation")) + || (error_str.contains("Invalid prompt") + && error_str.contains("violating our usage policy")) + || error_str + .to_lowercase() + .contains("request was rejected as a result of the safety system") + }, + kind: with_response(StatusClass::ContentPolicyViolation), + message: |mapping| prefixed(mapping, "ContentPolicyViolationError: "), + debug: true, + }, + Rule { + when: |mapping| { + contains_any( + &mapping.error_str, + &["invalid_encrypted_content", "could not be verified"], + ) + }, + kind: with_response(StatusClass::BadRequest), + message: |mapping| { + format!( + "{} - {}{ENCRYPTED_CONTENT_HELP}", + exception_provider(mapping), + message(mapping) + ) + }, + debug: true, + }, + Rule { + when: |mapping| { + mapping.error_str.contains("invalid_request_error") + && !mapping.error_str.contains("Incorrect API key provided") + }, + kind: with_response(StatusClass::BadRequest), + message: |mapping| prefixed(mapping, ""), + debug: true, + }, + Rule { + when: |mapping| { + contains_any( + &mapping.error_str, + &[ + "Web server is returning an unknown error", + "The server had an error processing your request.", + ], + ) + }, + kind: Kind::Status { + class: StatusClass::InternalServer, + response: ResponseChoice::Omitted, + }, + message: |mapping| prefixed(mapping, ""), + debug: false, + }, + Rule { + when: |mapping| mapping.error_str.contains("Request too large"), + kind: with_response(StatusClass::RateLimit), + message: |mapping| prefixed(mapping, "RateLimitError: "), + debug: true, + }, + Rule { + when: |mapping| { + mapping.error_str.contains("The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable") + }, + kind: with_response(StatusClass::Authentication), + message: |mapping| prefixed(mapping, "AuthenticationError: "), + debug: true, + }, + Rule { + when: |mapping| { + mapping + .error_str + .contains("Mistral API raised a streaming error") + }, + kind: Kind::Api { + status: ApiStatus::Fixed(500), + request_url: OPENAI_URL, + }, + message: |mapping| prefixed(mapping, ""), + debug: true, + }, + Rule { + when: |mapping| mapping.original.status.is_none(), + kind: Kind::ApiConnection, + message: |mapping| prefixed(mapping, "APIConnectionError: "), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[400, 422]), + kind: with_response(StatusClass::BadRequest), + message: |mapping| prefixed(mapping, ""), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[401]), + kind: with_response(StatusClass::Authentication), + message: |mapping| prefixed(mapping, "AuthenticationError: "), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[404]), + kind: with_response(StatusClass::NotFound), + message: |mapping| prefixed(mapping, "NotFoundError: "), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[408]), + kind: Kind::Timeout(None), + message: |mapping| prefixed(mapping, "Timeout Error: "), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[429]), + kind: with_response(StatusClass::RateLimit), + message: |mapping| prefixed(mapping, "RateLimitError: "), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[500]), + kind: with_response(StatusClass::InternalServer), + message: |mapping| prefixed(mapping, "InternalServerError: "), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[502]), + kind: with_response(StatusClass::BadGateway), + message: |mapping| prefixed(mapping, "BadGatewayError: "), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[503]), + kind: with_response(StatusClass::ServiceUnavailable), + message: |mapping| prefixed(mapping, "ServiceUnavailableError: "), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, &[504]), + kind: Kind::Timeout(Some(504)), + message: |mapping| prefixed(mapping, "Timeout Error: "), + debug: true, + }, + Rule { + when: |_| true, + kind: Kind::Api { + status: ApiStatus::Original, + request_url: DOCS_URL, + }, + message: |mapping| prefixed(mapping, "APIError: "), + debug: true, + }, +]; + +pub(super) fn map(mapping: &Mapping<'_>) -> Option { + apply(RULES, mapping) +} + +#[cfg(test)] +mod tests { + use super::super::testing::{context, failure, http, status, upstream, with_debug}; + use super::super::{ExceptionFamily, OriginalException, PublicKind}; + use super::*; + + fn mapped(provider: &str, original: &OriginalException) -> PublicFailure { + let context = context(provider, ExceptionFamily::OpenAiCompatible); + map(&Mapping::new(&context, original)).expect("the OpenAI table ends in a catch-all") + } + + fn kind(class: StatusClass, status_code: u16, body: &str) -> PublicKind { + status(class, upstream(status_code, body)) + } + + #[rstest::rstest] + #[case::rate_limit_phrase( + 400, + "rate limit reached", + failure( + kind(StatusClass::RateLimit, 400, "rate limit reached"), + "RateLimitError: MistralException - rate limit reached", + "mistral", + ) + )] + #[case::context_window( + 500, + "This model's maximum context length is 10", + with_debug(failure( + kind( + StatusClass::ContextWindowExceeded, + 500, + "This model's maximum context length is 10" + ), + "ContextWindowExceededError: MistralException - This model's maximum context length is 10", + "mistral", + )) + )] + #[case::model_not_found( + 400, + "invalid_request_error model_not_found", + with_debug(failure( + kind(StatusClass::NotFound, 400, "invalid_request_error model_not_found"), + "MistralException - invalid_request_error model_not_found", + "mistral", + )) + )] + #[case::timeout_occurred(400, "A timeout occurred", with_debug(failure( + PublicKind::Timeout { status: None }, + "MistralException - A timeout occurred", + "mistral", + )))] + #[case::content_policy_error_code( + 400, + "invalid_request_error content_policy_violation", + with_debug(failure( + kind( + StatusClass::ContentPolicyViolation, + 400, + "invalid_request_error content_policy_violation" + ), + "ContentPolicyViolationError: MistralException - invalid_request_error content_policy_violation", + "mistral", + )) + )] + #[case::content_policy_usage_policy( + 400, + "Invalid prompt violating our usage policy", + with_debug(failure( + kind( + StatusClass::ContentPolicyViolation, + 400, + "Invalid prompt violating our usage policy" + ), + "ContentPolicyViolationError: MistralException - Invalid prompt violating our usage policy", + "mistral", + )) + )] + #[case::content_policy_safety_system( + 400, + "Request was rejected as a result of the safety system", + with_debug(failure( + kind( + StatusClass::ContentPolicyViolation, + 400, + "Request was rejected as a result of the safety system" + ), + "ContentPolicyViolationError: MistralException - Request was rejected as a result of the safety system", + "mistral", + )) + )] + #[case::encrypted_content(400, "invalid_encrypted_content", with_debug(failure( + kind(StatusClass::BadRequest, 400, "invalid_encrypted_content"), + &format!("MistralException - invalid_encrypted_content{ENCRYPTED_CONTENT_HELP}"), + "mistral", + )))] + #[case::unverifiable_content(400, "could not be verified", with_debug(failure( + kind(StatusClass::BadRequest, 400, "could not be verified"), + &format!("MistralException - could not be verified{ENCRYPTED_CONTENT_HELP}"), + "mistral", + )))] + #[case::invalid_request( + 429, + "invalid_request_error bad field", + with_debug(failure( + kind(StatusClass::BadRequest, 429, "invalid_request_error bad field"), + "MistralException - invalid_request_error bad field", + "mistral", + )) + )] + #[case::unknown_server_error( + 400, + "Web server is returning an unknown error", + failure( + status(StatusClass::InternalServer, None), + "MistralException - Web server is returning an unknown error", + "mistral", + ) + )] + #[case::server_had_an_error( + 400, + "The server had an error processing your request.", + failure( + status(StatusClass::InternalServer, None), + "MistralException - The server had an error processing your request.", + "mistral", + ) + )] + #[case::request_too_large( + 400, + "Request too large", + with_debug(failure( + kind(StatusClass::RateLimit, 400, "Request too large"), + "RateLimitError: MistralException - Request too large", + "mistral", + )) + )] + #[case::missing_client_api_key( + 400, + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable", + with_debug(failure( + kind( + StatusClass::Authentication, + 400, + "The api_key client option must be set either by passing api_key to the client or by setting the OPENAI_API_KEY environment variable", + ), + "AuthenticationError: MistralException - The api_key client option must be set either by passing api_key to the client or by setting the MISTRAL_API_KEY environment variable", + "mistral", + )) + )] + #[case::mistral_streaming_error(400, "Mistral API raised a streaming error", with_debug(failure( + PublicKind::Api { status: 500, request_url: OPENAI_URL }, + "MistralException - Mistral API raised a streaming error", + "mistral", + )))] + fn each_text_rule_maps_by_the_body( + #[case] status_code: u16, + #[case] body: &str, + #[case] expected: PublicFailure, + ) { + assert_eq!(mapped("mistral", &http(status_code, body)), expected); + } + + #[rstest::rstest] + #[case::bad_request( + 400, + kind(StatusClass::BadRequest, 400, "rejected"), + "MistralException - rejected" + )] + #[case::unprocessable( + 422, + kind(StatusClass::BadRequest, 422, "rejected"), + "MistralException - rejected" + )] + #[case::authentication( + 401, + kind(StatusClass::Authentication, 401, "rejected"), + "AuthenticationError: MistralException - rejected" + )] + #[case::not_found( + 404, + kind(StatusClass::NotFound, 404, "rejected"), + "NotFoundError: MistralException - rejected" + )] + #[case::request_timeout(408, PublicKind::Timeout { status: None }, "Timeout Error: MistralException - rejected")] + #[case::rate_limited( + 429, + kind(StatusClass::RateLimit, 429, "rejected"), + "RateLimitError: MistralException - rejected" + )] + #[case::internal_server( + 500, + kind(StatusClass::InternalServer, 500, "rejected"), + "InternalServerError: MistralException - rejected" + )] + #[case::bad_gateway( + 502, + kind(StatusClass::BadGateway, 502, "rejected"), + "BadGatewayError: MistralException - rejected" + )] + #[case::service_unavailable( + 503, + kind(StatusClass::ServiceUnavailable, 503, "rejected"), + "ServiceUnavailableError: MistralException - rejected" + )] + #[case::gateway_timeout(504, PublicKind::Timeout { status: Some(504) }, "Timeout Error: MistralException - rejected")] + #[case::any_other_status(409, PublicKind::Api { status: 409, request_url: DOCS_URL }, "APIError: MistralException - rejected")] + fn each_status_rule_maps_by_the_status( + #[case] status_code: u16, + #[case] kind: PublicKind, + #[case] message: &str, + ) { + assert_eq!( + mapped("mistral", &http(status_code, "rejected")), + with_debug(failure(kind, message, "mistral")) + ); + } + + #[test] + fn a_failure_without_a_status_is_a_connection_error() { + let original = OriginalException::Response { + message: "invalid OCR response field: pages".into(), + }; + assert_eq!( + mapped("mistral", &original), + with_debug(failure( + PublicKind::ApiConnection, + "APIConnectionError: MistralException - invalid OCR response field: pages", + "mistral" + )) + ); + } + + #[rstest::rstest] + #[case::rate_limit_before_context_window( + 400, + "rate limit and This model's maximum context length is 10", + kind( + StatusClass::RateLimit, + 400, + "rate limit and This model's maximum context length is 10" + ), + "RateLimitError: MistralException - rate limit and This model's maximum context length is 10", + false + )] + #[case::context_window_before_content_policy( + 400, + "This model's maximum context length is 10 invalid_request_error content_policy_violation", + kind( + StatusClass::ContextWindowExceeded, + 400, + "This model's maximum context length is 10 invalid_request_error content_policy_violation" + ), + "ContextWindowExceededError: MistralException - This model's maximum context length is 10 invalid_request_error content_policy_violation", + true + )] + #[case::model_not_found_before_invalid_request( + 400, + "invalid_request_error model_not_found", + kind(StatusClass::NotFound, 400, "invalid_request_error model_not_found"), + "MistralException - invalid_request_error model_not_found", + true + )] + #[case::timeout_before_invalid_request( + 400, + "A timeout occurred invalid_request_error", + PublicKind::Timeout { status: None }, + "MistralException - A timeout occurred invalid_request_error", + true + )] + #[case::content_policy_before_invalid_request( + 400, + "invalid_request_error content_policy_violation", + kind( + StatusClass::ContentPolicyViolation, + 400, + "invalid_request_error content_policy_violation" + ), + "ContentPolicyViolationError: MistralException - invalid_request_error content_policy_violation", + true + )] + #[case::invalid_request_with_a_bad_key_falls_to_the_status( + 401, + "invalid_request_error Incorrect API key provided", + kind( + StatusClass::Authentication, + 401, + "invalid_request_error Incorrect API key provided" + ), + "AuthenticationError: MistralException - invalid_request_error Incorrect API key provided", + true + )] + #[case::text_rules_before_status( + 429, + "invalid_request_error bad field", + kind(StatusClass::BadRequest, 429, "invalid_request_error bad field"), + "MistralException - invalid_request_error bad field", + true + )] + #[case::echoed_429_is_not_a_rate_limit( + 400, + "token 429 in the prompt", + kind(StatusClass::BadRequest, 400, "token 429 in the prompt"), + "MistralException - token 429 in the prompt", + true + )] + fn the_earlier_rule_wins_when_two_apply( + #[case] status_code: u16, + #[case] body: &str, + #[case] kind: PublicKind, + #[case] message: &str, + #[case] debug: bool, + ) { + let expected = failure(kind, message, "mistral"); + assert_eq!( + mapped("mistral", &http(status_code, body)), + if debug { + with_debug(expected) + } else { + expected + } + ); + } + + #[rstest::rstest] + #[case::provider_names_replace_openai( + "azure_ai", + "OPENAI said openai.OpenAIError", + "Azure_aiException - AZURE_AI said azure_ai.azure_aiError" + )] + #[case::openai_keeps_its_own_name("openai", "rejected", "OpenAIException - rejected")] + fn the_message_names_the_provider( + #[case] provider: &str, + #[case] body: &str, + #[case] message: &str, + ) { + assert_eq!( + mapped(provider, &http(400, body)), + with_debug(failure( + kind(StatusClass::BadRequest, 400, body), + message, + provider + )) + ); + } +} diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs new file mode 100644 index 00000000000..d64868c1606 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/original.rs @@ -0,0 +1,49 @@ +use super::public::StatusClass; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum LocalClass { + ValueError, + FileNotFound, + OsError, +} + +/// A route failure in the shape Python's `exception_type` receives it, before any public +/// class is chosen. +#[derive(Clone, Debug, PartialEq)] +pub enum OriginalException { + Http { + status: u16, + body: String, + headers: Vec<(String, String)>, + }, + Connection { + message: String, + }, + Timeout { + timeout_seconds: Option, + elapsed_seconds: Option, + }, + Response { + message: String, + }, + Local { + class: LocalClass, + message: String, + }, + /// A failure Python raises as a public LiteLLM exception itself, which `exception_type` + /// hands back unchanged. + Public { + class: StatusClass, + message: String, + }, +} + +/// Which of the provider-specific mappers in `exception_type` a route's provider uses. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum ExceptionFamily { + OpenAiCompatible, + VertexAi, + Cohere, + #[default] + Other, +} diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/public.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/public.rs new file mode 100644 index 00000000000..a7319c1287b --- /dev/null +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/public.rs @@ -0,0 +1,262 @@ +use serde::Serialize; + +/// The public LiteLLM classes built from a status code alone: every one takes the same +/// constructor arguments. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, strum::EnumIter, strum::IntoStaticStr)] +#[serde(rename_all = "snake_case")] +#[strum(serialize_all = "snake_case")] +pub enum StatusClass { + BadRequest, + Authentication, + PermissionDenied, + NotFound, + RateLimit, + ContextWindowExceeded, + ContentPolicyViolation, + InternalServer, + BadGateway, + ServiceUnavailable, + UnsupportedParams, +} + +impl StatusClass { + /// The `status_code` the Python class sets on itself. + pub const fn status_code(self) -> u16 { + match self { + Self::BadRequest + | Self::ContextWindowExceeded + | Self::ContentPolicyViolation + | Self::UnsupportedParams => 400, + Self::Authentication => 401, + Self::PermissionDenied => 403, + Self::NotFound => 404, + Self::RateLimit => 429, + Self::InternalServer => 500, + Self::BadGateway => 502, + Self::ServiceUnavailable => 503, + } + } +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct UpstreamResponse { + pub status: u16, + pub body: String, + pub headers: Vec<(String, String)>, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct HttpStub { + pub status: u16, + pub method: &'static str, + pub url: &'static str, + pub content: Option, +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum ResponseArg { + Upstream(UpstreamResponse), + Stub(HttpStub), +} + +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum PublicKind { + Status { + status_class: StatusClass, + response: Option, + }, + Timeout { + status: Option, + }, + ApiConnection, + Api { + status: u16, + request_url: &'static str, + }, +} + +/// Constructor arguments for the public LiteLLM exception, as `exception_type` passes them. +#[derive(Clone, Debug, PartialEq, Eq, Serialize)] +pub struct PublicFailure { + pub kind: PublicKind, + pub message: String, + pub model: String, + pub llm_provider: Option, + pub litellm_debug_info: Option, + pub litellm_response_headers: Option>, + pub print_banner: bool, +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeSet; + use std::path::PathBuf; + + use serde_json::Value; + use strum::IntoEnumIterator; + + use super::*; + + const REGENERATE: &str = "LITELLM_REGENERATE_PUBLIC_FAILURE_FIXTURES"; + + fn fixture_directory() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../../tests/test_litellm/rust_bridge/fixtures/public_failures") + } + + fn upstream(status: u16) -> ResponseArg { + ResponseArg::Upstream(UpstreamResponse { + status, + body: r#"{"message": "rejected"}"#.into(), + headers: vec![("retry-after".into(), "7".into())], + }) + } + + fn status_response(class: StatusClass) -> Option { + match class { + StatusClass::Authentication => None, + StatusClass::PermissionDenied => Some(ResponseArg::Stub(HttpStub { + status: 403, + method: "POST", + url: " https://cloud.google.com/vertex-ai/", + content: None, + })), + StatusClass::InternalServer => Some(ResponseArg::Stub(HttpStub { + status: 500, + method: "completion", + url: "https://github.com/BerriAI/litellm", + content: Some("upstream text".into()), + })), + class => Some(upstream(class.status_code())), + } + } + + fn failure(kind: PublicKind, name: &str) -> PublicFailure { + let headers = matches!( + &kind, + PublicKind::Status { + response: Some(ResponseArg::Upstream(_)), + .. + } + ); + PublicFailure { + kind, + message: format!("MistralException - {name}"), + model: "ocr-model".into(), + llm_provider: Some("mistral".into()), + litellm_debug_info: Some("\nModel: ocr-model".into()), + litellm_response_headers: headers.then(|| vec![("retry-after".into(), "7".into())]), + print_banner: false, + } + } + + /// One payload per public class the constructor can build; `test_failures.py` reads the + /// same files, so a shape change on either side fails there or here. + fn fixtures() -> Vec<(String, PublicFailure)> { + let statuses = StatusClass::iter().map(|class| { + let name: &'static str = class.into(); + let name = format!("status_{name}"); + let built = failure( + PublicKind::Status { + status_class: class, + response: status_response(class), + }, + &name, + ); + (name, built) + }); + let others = [ + ( + "timeout_with_status", + PublicFailure { + print_banner: true, + ..failure( + PublicKind::Timeout { status: Some(504) }, + "timeout_with_status", + ) + }, + ), + ( + "timeout_without_status", + PublicFailure { + litellm_debug_info: None, + ..failure( + PublicKind::Timeout { status: None }, + "timeout_without_status", + ) + }, + ), + ( + "api_connection", + PublicFailure { + llm_provider: None, + ..failure(PublicKind::ApiConnection, "api_connection") + }, + ), + ( + "api", + failure( + PublicKind::Api { + status: 409, + request_url: "https://docs.litellm.ai/docs", + }, + "api", + ), + ), + ] + .map(|(name, built)| (name.to_string(), built)); + statuses.chain(others).collect() + } + + #[test] + fn serialized_payloads_match_the_golden_fixtures_python_reads() { + let directory = fixture_directory(); + let regenerate = std::env::var_os(REGENERATE).is_some(); + let expected = fixtures(); + for (name, built) in &expected { + let path = directory.join(format!("{name}.json")); + let serialized = serde_json::to_value(built).unwrap(); + if regenerate { + std::fs::create_dir_all(&directory).unwrap(); + std::fs::write( + &path, + format!("{}\n", serde_json::to_string_pretty(&serialized).unwrap()), + ) + .unwrap(); + } + let golden: Value = + serde_json::from_str(&std::fs::read_to_string(&path).unwrap()).unwrap(); + assert_eq!(serialized, golden, "{name}; set {REGENERATE}=1 to rewrite"); + } + let on_disk: BTreeSet = std::fs::read_dir(&directory) + .unwrap() + .map(|entry| entry.unwrap().file_name().to_string_lossy().into_owned()) + .collect(); + let generated: BTreeSet = expected + .iter() + .map(|(name, _)| format!("{name}.json")) + .collect(); + assert_eq!(on_disk, generated); + } + + #[rstest::rstest] + #[case(StatusClass::BadRequest, 400)] + #[case(StatusClass::Authentication, 401)] + #[case(StatusClass::PermissionDenied, 403)] + #[case(StatusClass::NotFound, 404)] + #[case(StatusClass::RateLimit, 429)] + #[case(StatusClass::ContextWindowExceeded, 400)] + #[case(StatusClass::ContentPolicyViolation, 400)] + #[case(StatusClass::InternalServer, 500)] + #[case(StatusClass::BadGateway, 502)] + #[case(StatusClass::ServiceUnavailable, 503)] + #[case(StatusClass::UnsupportedParams, 400)] + fn status_codes_are_the_ones_the_python_classes_set( + #[case] class: StatusClass, + #[case] status: u16, + ) { + assert_eq!(class.status_code(), status); + } +} diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/rules.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/rules.rs new file mode 100644 index 00000000000..5df8ad991b5 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/rules.rs @@ -0,0 +1,403 @@ +use std::sync::LazyLock; + +use fancy_regex::Regex; +use serde_json::Value; + +use super::Mapping; +use super::public::{HttpStub, PublicFailure, PublicKind, ResponseArg, StatusClass}; + +const GITHUB_URL: &str = "https://github.com/BerriAI/litellm"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ResponseChoice { + Omitted, + Provider, + Stub { status: u16, url: &'static str }, + InternalServerStub, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ApiStatus { + Fixed(u16), + Original, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum Kind { + Status { + class: StatusClass, + response: ResponseChoice, + }, + Timeout(Option), + ApiConnection, + Api { + status: ApiStatus, + request_url: &'static str, + }, +} + +/// One branch of a Python `_map_*_exception` function: when it applies, the class it +/// raises, the message it builds, and whether it passes `litellm_debug_info`. +pub(super) struct Rule { + pub(super) when: fn(&Mapping<'_>) -> bool, + pub(super) kind: Kind, + pub(super) message: fn(&Mapping<'_>) -> String, + pub(super) debug: bool, +} + +/// The first rule that applies decides the failure, as the `if`/`elif` chain does in Python. +pub(super) fn apply(rules: &[Rule], mapping: &Mapping<'_>) -> Option { + rules + .iter() + .find(|rule| (rule.when)(mapping)) + .map(|rule| rule.build(mapping)) +} + +impl Rule { + fn build(&self, mapping: &Mapping<'_>) -> PublicFailure { + let kind = match self.kind { + Kind::Status { class, response } => PublicKind::Status { + status_class: class, + response: response.resolve(mapping), + }, + Kind::Timeout(status) => PublicKind::Timeout { status }, + Kind::ApiConnection => PublicKind::ApiConnection, + Kind::Api { + status, + request_url, + } => PublicKind::Api { + status: match status { + ApiStatus::Fixed(status) => status, + ApiStatus::Original => mapping.original.status.unwrap_or(500), + }, + request_url, + }, + }; + PublicFailure { + kind, + message: (self.message)(mapping), + model: mapping.context.model.clone(), + llm_provider: mapping.context.custom_llm_provider.clone(), + litellm_debug_info: self.debug.then(|| mapping.extra_information.clone()), + litellm_response_headers: None, + print_banner: false, + } + } +} + +impl ResponseChoice { + fn resolve(self, mapping: &Mapping<'_>) -> Option { + match self { + Self::Omitted => None, + Self::Provider => mapping.original.response.clone().map(ResponseArg::Upstream), + Self::Stub { status, url } => Some(ResponseArg::Stub(HttpStub { + status, + method: "POST", + url, + content: None, + })), + Self::InternalServerStub => Some(ResponseArg::Stub(HttpStub { + status: 500, + method: "completion", + url: GITHUB_URL, + content: Some(mapping.original.message.clone()), + })), + } + } +} + +pub(super) fn contains_any(text: &str, markers: &[&str]) -> bool { + markers.iter().any(|marker| text.contains(marker)) +} + +static STANDALONE_429: LazyLock = + LazyLock::new(|| Regex::new(r"\b429\b").expect("valid regex")); +static RATE_LIMIT_PHRASE: LazyLock = + LazyLock::new(|| Regex::new(r"rate[\s_\-]*limit").expect("valid regex")); + +/// `ExceptionCheckers.is_error_str_rate_limit`. +pub(super) fn is_rate_limit(error_str: &str, status: Option) -> bool { + if STANDALONE_429.is_match(error_str).unwrap_or(false) && status == Some(429) { + return true; + } + let lower = error_str.to_lowercase(); + RATE_LIMIT_PHRASE.is_match(&lower).unwrap_or(false) + || lower.contains("service tier capacity exceeded") +} + +/// `ExceptionCheckers.is_error_str_context_window_exceeded`. +pub(super) fn is_context_window_exceeded(error_str: &str) -> bool { + let lower = error_str.to_lowercase(); + if lower.contains("string_above_max_length") { + return false; + } + if lower.contains("invalid 'user'") && lower.contains("string too long") { + return false; + } + contains_any( + &lower, + &[ + "exceed context limit", + "this model's maximum context length is", + "string too long. expected a string with maximum length", + "model's maximum context limit", + "is longer than the model's context length", + "input tokens exceed the configured limit", + "`inputs` tokens + `max_new_tokens` must be", + "exceeds the available context size", + "exceeds the maximum number of tokens allowed", + ], + ) || (lower.contains("current length is") && lower.contains("while limit is")) + || (lower.contains("maximum input length is") && lower.contains("tokens")) +} + +/// The integer `error.code` of a JSON error body, read the way Python's `int()` would. +pub(super) fn body_error_code(error_str: &str) -> Option { + let body: Value = serde_json::from_str(error_str).ok()?; + let Some(Value::Object(error)) = body.as_object()?.get("error") else { + return None; + }; + match error.get("code")? { + Value::Number(number) => number + .as_i64() + .or_else(|| number.as_f64().map(|value| value.trunc() as i64)), + Value::String(code) => code.trim().replace('_', "").parse().ok(), + Value::Bool(flag) => Some(i64::from(*flag)), + _ => None, + } +} + +#[cfg(test)] +mod tests { + use super::super::testing::{context, failure, http}; + use super::super::{ExceptionFamily, OriginalException, UpstreamResponse}; + use super::*; + + fn first_marker(mapping: &Mapping<'_>) -> bool { + mapping.error_str.contains("first") + } + + fn always(_: &Mapping<'_>) -> bool { + true + } + + fn text(mapping: &Mapping<'_>) -> String { + format!("seen {}", mapping.error_str) + } + + const ORDERED: &[Rule] = &[ + Rule { + when: first_marker, + kind: Kind::Status { + class: StatusClass::NotFound, + response: ResponseChoice::Omitted, + }, + message: text, + debug: false, + }, + Rule { + when: always, + kind: Kind::ApiConnection, + message: text, + debug: true, + }, + ]; + + fn apply_one(kind: Kind, debug: bool, original: &OriginalException) -> Option { + let context = context("mistral", ExceptionFamily::OpenAiCompatible); + let mapping = Mapping::new(&context, original); + apply( + &[Rule { + when: always, + kind, + message: text, + debug, + }], + &mapping, + ) + } + + #[rstest::rstest] + #[case::earlier_rule_wins("first and second", failure( + PublicKind::Status { status_class: StatusClass::NotFound, response: None }, + "seen first and second", + "mistral", + ))] + #[case::later_rule_when_the_earlier_does_not_apply("second", PublicFailure { + litellm_debug_info: Some("\nModel: ocr-model\nMessages: `None`".into()), + ..failure(PublicKind::ApiConnection, "seen second", "mistral") + })] + fn the_first_applicable_rule_decides(#[case] body: &str, #[case] expected: PublicFailure) { + let context = context("mistral", ExceptionFamily::OpenAiCompatible); + let original = http(400, body); + assert_eq!( + apply(ORDERED, &Mapping::new(&context, &original)), + Some(expected) + ); + } + + #[test] + fn no_applicable_rule_leaves_the_failure_to_the_caller() { + let context = context("mistral", ExceptionFamily::OpenAiCompatible); + let original = http(400, "second"); + assert_eq!( + apply(&ORDERED[..1], &Mapping::new(&context, &original)), + None + ); + } + + #[rstest::rstest] + #[case::omitted(ResponseChoice::Omitted, None)] + #[case::provider(ResponseChoice::Provider, Some(ResponseArg::Upstream(UpstreamResponse { + status: 400, + body: "body".into(), + headers: vec![("retry-after".into(), "7".into())], + })))] + #[case::stub( + ResponseChoice::Stub { status: 429, url: "https://stub.test" }, + Some(ResponseArg::Stub(HttpStub { status: 429, method: "POST", url: "https://stub.test", content: None })) + )] + #[case::internal_server_stub( + ResponseChoice::InternalServerStub, + Some(ResponseArg::Stub(HttpStub { + status: 500, + method: "completion", + url: GITHUB_URL, + content: Some("body".into()), + })) + )] + fn response_choices_resolve_against_the_original( + #[case] response: ResponseChoice, + #[case] expected: Option, + ) { + let built = apply_one( + Kind::Status { + class: StatusClass::BadRequest, + response, + }, + false, + &http(400, "body"), + ) + .unwrap(); + assert_eq!( + built.kind, + PublicKind::Status { + status_class: StatusClass::BadRequest, + response: expected, + } + ); + } + + #[rstest::rstest] + #[case::fixed(ApiStatus::Fixed(500), http(409, "body"), 500)] + #[case::original(ApiStatus::Original, http(409, "body"), 409)] + #[case::original_without_a_status( + ApiStatus::Original, + OriginalException::Response { message: "body".into() }, + 500 + )] + fn api_status_is_fixed_or_the_originals( + #[case] status: ApiStatus, + #[case] original: OriginalException, + #[case] expected: u16, + ) { + let built = apply_one( + Kind::Api { + status, + request_url: "https://api.test", + }, + false, + &original, + ) + .unwrap(); + assert_eq!( + built, + failure( + PublicKind::Api { + status: expected, + request_url: "https://api.test" + }, + "seen body", + "mistral" + ) + ); + } + + #[rstest::rstest] + #[case::with_debug(true, Some("\nModel: ocr-model\nMessages: `None`"))] + #[case::without_debug(false, None)] + fn debug_rules_carry_the_extra_information( + #[case] debug: bool, + #[case] expected: Option<&str>, + ) { + let built = apply_one(Kind::Timeout(Some(504)), debug, &http(504, "body")).unwrap(); + assert_eq!( + built, + PublicFailure { + litellm_debug_info: expected.map(str::to_string), + ..failure( + PublicKind::Timeout { status: Some(504) }, + "seen body", + "mistral" + ) + } + ); + } + + #[rstest::rstest] + #[case::standalone_429_with_429_status("got 429 back", Some(429), true)] + #[case::standalone_429_with_other_status("got 429 back", Some(400), false)] + #[case::embedded_429("token4290", Some(429), false)] + #[case::phrase_spaced("Rate Limit reached", None, true)] + #[case::phrase_underscored("rate_limit", None, true)] + #[case::phrase_hyphenated("rate-limit", None, true)] + #[case::service_tier("Service tier capacity exceeded", None, true)] + #[case::unrelated("rejected", Some(429), false)] + fn rate_limit_detection( + #[case] text: &str, + #[case] status: Option, + #[case] expected: bool, + ) { + assert_eq!(is_rate_limit(text, status), expected); + } + + #[rstest::rstest] + #[case::exceed_context_limit("Exceed context limit", true)] + #[case::maximum_context_length("This model's maximum context length is 10", true)] + #[case::string_too_long("string too long. Expected a string with maximum length 5", true)] + #[case::maximum_context_limit("the model's maximum context limit", true)] + #[case::longer_than_context("prompt is longer than the model's context length", true)] + #[case::configured_limit("input tokens exceed the configured limit", true)] + #[case::max_new_tokens("`inputs` tokens + `max_new_tokens` must be <= 10", true)] + #[case::available_context("exceeds the available context size", true)] + #[case::maximum_tokens("exceeds the maximum number of tokens allowed", true)] + #[case::current_and_limit("current length is 9 while limit is 8", true)] + #[case::current_without_limit("current length is 9", false)] + #[case::maximum_input_tokens("maximum input length is 8 tokens", true)] + #[case::maximum_input_without_tokens("maximum input length is 8", false)] + #[case::string_above_max_length_wins("string_above_max_length exceed context limit", false)] + #[case::user_field_is_not_context( + "invalid 'user': string too long. expected a string with maximum length", + false + )] + #[case::unrelated("rejected", false)] + fn context_window_detection(#[case] text: &str, #[case] expected: bool) { + assert_eq!(is_context_window_exceeded(text), expected); + } + + #[rstest::rstest] + #[case::integer(r#"{"error": {"code": 429}}"#, Some(429))] + #[case::float(r#"{"error": {"code": 429.9}}"#, Some(429))] + #[case::string(r#"{"error": {"code": " 4_29 "}}"#, Some(429))] + #[case::boolean(r#"{"error": {"code": true}}"#, Some(1))] + #[case::unparseable_string(r#"{"error": {"code": "slow"}}"#, None)] + #[case::null(r#"{"error": {"code": null}}"#, None)] + #[case::no_code(r#"{"error": {}}"#, None)] + #[case::error_not_an_object(r#"{"error": "429"}"#, None)] + #[case::no_error(r#"{"code": 429}"#, None)] + #[case::not_an_object("[429]", None)] + #[case::not_json("429", None)] + fn body_error_code_reads_the_nested_code(#[case] body: &str, #[case] expected: Option) { + assert_eq!(body_error_code(body), expected); + } +} diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/status.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/status.rs new file mode 100644 index 00000000000..3d817fe9ae2 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/status.rs @@ -0,0 +1,153 @@ +use super::public::{PublicFailure, StatusClass}; +use super::rules::{ApiStatus, Kind, ResponseChoice, Rule, apply}; +use super::{DOCS_URL, Mapping}; + +const fn with_response(class: StatusClass) -> Kind { + Kind::Status { + class, + response: ResponseChoice::Provider, + } +} + +fn message(mapping: &Mapping<'_>) -> String { + format!("{} - {}", mapping.exception_provider, mapping.error_str) +} + +fn status(mapping: &Mapping<'_>) -> u16 { + mapping.original.status.unwrap_or_default() +} + +/// `_map_exception_by_status`, the fallback for a provider error no provider mapper claimed. +const RULES: &[Rule] = &[ + Rule { + when: |mapping| status(mapping) == 401, + kind: with_response(StatusClass::Authentication), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) == 403, + kind: with_response(StatusClass::PermissionDenied), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) == 404, + kind: with_response(StatusClass::NotFound), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) == 408, + kind: Kind::Timeout(None), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) == 429, + kind: with_response(StatusClass::RateLimit), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) == 500, + kind: with_response(StatusClass::InternalServer), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) == 502, + kind: with_response(StatusClass::BadGateway), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) == 503, + kind: with_response(StatusClass::ServiceUnavailable), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) == 504, + kind: Kind::Timeout(Some(504)), + message, + debug: true, + }, + Rule { + when: |mapping| status(mapping) < 500, + kind: with_response(StatusClass::BadRequest), + message, + debug: true, + }, + Rule { + when: |_| true, + kind: Kind::Api { + status: ApiStatus::Original, + request_url: DOCS_URL, + }, + message, + debug: true, + }, +]; + +/// Only a real provider status of 400 or more reaches the table; a status the HTTP handler +/// synthesized for a failure without a response does not. +pub(super) fn map(mapping: &Mapping<'_>) -> Option { + let status = mapping.original.status?; + if status < 400 || mapping.original.status_is_synthesized { + return None; + } + apply(RULES, mapping) +} + +#[cfg(test)] +mod tests { + use super::super::testing::{context, failure, http, upstream, with_debug}; + use super::super::{ExceptionFamily, OriginalException, PublicKind}; + use super::*; + + fn mapped(original: &OriginalException) -> Option { + let context = context("reducto", ExceptionFamily::Other); + map(&Mapping::new(&context, original)) + } + + fn classified(class: StatusClass, status_code: u16) -> PublicKind { + PublicKind::Status { + status_class: class, + response: upstream(status_code, "rejected"), + } + } + + #[rstest::rstest] + #[case::authentication(401, classified(StatusClass::Authentication, 401))] + #[case::permission_denied(403, classified(StatusClass::PermissionDenied, 403))] + #[case::not_found(404, classified(StatusClass::NotFound, 404))] + #[case::request_timeout(408, PublicKind::Timeout { status: None })] + #[case::rate_limited(429, classified(StatusClass::RateLimit, 429))] + #[case::internal_server(500, classified(StatusClass::InternalServer, 500))] + #[case::bad_gateway(502, classified(StatusClass::BadGateway, 502))] + #[case::service_unavailable(503, classified(StatusClass::ServiceUnavailable, 503))] + #[case::gateway_timeout(504, PublicKind::Timeout { status: Some(504) })] + #[case::lowest_client_error(400, classified(StatusClass::BadRequest, 400))] + #[case::other_client_error(409, classified(StatusClass::BadRequest, 409))] + #[case::highest_client_error(499, classified(StatusClass::BadRequest, 499))] + #[case::other_server_error(501, PublicKind::Api { status: 501, request_url: DOCS_URL })] + fn every_mapped_status_and_the_fallback(#[case] status_code: u16, #[case] kind: PublicKind) { + assert_eq!( + mapped(&http(status_code, "rejected")), + Some(with_debug(failure( + kind, + "ReductoException - rejected", + "reducto" + ))) + ); + } + + #[rstest::rstest] + #[case::below_client_errors(http(399, "rejected"))] + #[case::synthesized(OriginalException::Connection { message: "refused".into() })] + #[case::no_status(OriginalException::Response { message: "bad body".into() })] + fn failures_the_table_does_not_claim(#[case] original: OriginalException) { + assert_eq!(mapped(&original), None); + } +} diff --git a/litellm-rust/crates/core-utils/src/exception_mapping_utils/vertex_ai.rs b/litellm-rust/crates/core-utils/src/exception_mapping_utils/vertex_ai.rs new file mode 100644 index 00000000000..0fa8c19c4f6 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/exception_mapping_utils/vertex_ai.rs @@ -0,0 +1,574 @@ +use super::public::{PublicFailure, PublicKind, ResponseArg, StatusClass, UpstreamResponse}; +use super::rules::{ + Kind, ResponseChoice, Rule, apply, body_error_code, contains_any, is_context_window_exceeded, +}; +use super::{Mapping, python_capitalize}; + +const VERTEX_URL: &str = "https://cloud.google.com/vertex-ai/"; +const VERTEX_URL_WITH_SPACE: &str = " https://cloud.google.com/vertex-ai/"; + +const QUOTA_MARKERS: &[&str] = &[ + "429 Quota exceeded", + "Quota exceeded for", + "Resource exhausted", + "IndexError: list index out of range", + "429 Unable to submit request because the service is temporarily out of capacity.", +]; + +const fn stubbed(class: StatusClass, status: u16, url: &'static str) -> Kind { + Kind::Status { + class, + response: ResponseChoice::Stub { status, url }, + } +} + +const fn bare(class: StatusClass) -> Kind { + Kind::Status { + class, + response: ResponseChoice::Omitted, + } +} + +/// `{Provider}Exception{label} - {error_str}` with Python's `str.capitalize()`. +fn capitalized(mapping: &Mapping<'_>, label: &str) -> String { + format!( + "{}Exception{label} - {}", + python_capitalize(mapping.provider), + mapping.error_str + ) +} + +/// `litellm.{Class}: {provider}Exception - {error_str}` with the provider as given. +fn litellm_prefixed(mapping: &Mapping<'_>, class: &str) -> String { + format!( + "litellm.{class}: {}Exception - {}", + mapping.provider, mapping.error_str + ) +} + +fn status_is(mapping: &Mapping<'_>, status: u16) -> bool { + mapping.original.status == Some(status) +} + +/// `_map_vertex_exception`, in its branch order. A failure no rule claims falls through +/// to the status table. +const RULES: &[Rule] = &[ + Rule { + when: |mapping| { + contains_any( + &mapping.error_str, + &[ + "Vertex AI API has not been used in project", + "Unable to find your project", + ], + ) + }, + kind: stubbed(StatusClass::BadRequest, 400, VERTEX_URL_WITH_SPACE), + message: |mapping| litellm_prefixed(mapping, "BadRequestError"), + debug: true, + }, + Rule { + when: |mapping| { + mapping + .error_str + .contains("400 Request payload size exceeds") + }, + kind: bare(StatusClass::ContextWindowExceeded), + message: |mapping| capitalized(mapping, ""), + debug: false, + }, + Rule { + when: |mapping| is_context_window_exceeded(&mapping.error_str), + kind: bare(StatusClass::ContextWindowExceeded), + message: |mapping| format!("ContextWindowExceededError: {}", capitalized(mapping, "")), + debug: true, + }, + Rule { + when: |mapping| { + contains_any( + &mapping.error_str, + &["None Unknown Error.", "Content has no parts."], + ) + }, + kind: Kind::Status { + class: StatusClass::InternalServer, + response: ResponseChoice::InternalServerStub, + }, + message: |mapping| litellm_prefixed(mapping, "InternalServerError"), + debug: true, + }, + Rule { + when: |mapping| mapping.error_str.contains("API key not valid."), + kind: bare(StatusClass::Authentication), + message: |mapping| capitalized(mapping, ""), + debug: true, + }, + Rule { + when: |mapping| mapping.error_str.contains("403"), + kind: stubbed(StatusClass::BadRequest, 403, VERTEX_URL_WITH_SPACE), + message: |mapping| capitalized(mapping, " BadRequestError"), + debug: true, + }, + Rule { + when: |mapping| { + contains_any( + &mapping.error_str, + &[ + "The response was blocked.", + "Output blocked by content filtering policy", + ], + ) + }, + kind: stubbed( + StatusClass::ContentPolicyViolation, + 400, + VERTEX_URL_WITH_SPACE, + ), + message: |mapping| capitalized(mapping, " ContentPolicyViolationError"), + debug: true, + }, + Rule { + when: |mapping| { + contains_any(&mapping.error_str, QUOTA_MARKERS) + || (mapping + .original + .status + .is_some_and(|status| (500..600).contains(&status)) + && body_error_code(&mapping.error_str) == Some(429)) + }, + kind: stubbed(StatusClass::RateLimit, 429, VERTEX_URL_WITH_SPACE), + message: |mapping| litellm_prefixed(mapping, "RateLimitError"), + debug: true, + }, + Rule { + when: |mapping| { + contains_any( + &mapping.error_str, + &["500 Internal Server Error", "The model is overloaded."], + ) + }, + kind: bare(StatusClass::InternalServer), + message: |mapping| litellm_prefixed(mapping, "InternalServerError"), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, 400), + kind: stubbed(StatusClass::BadRequest, 400, VERTEX_URL), + message: |mapping| capitalized(mapping, " BadRequestError"), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, 401), + kind: bare(StatusClass::Authentication), + message: |mapping| capitalized(mapping, ""), + debug: false, + }, + Rule { + when: |mapping| status_is(mapping, 403), + kind: stubbed(StatusClass::PermissionDenied, 403, VERTEX_URL), + message: |mapping| capitalized(mapping, ""), + debug: false, + }, + Rule { + when: |mapping| status_is(mapping, 404), + kind: bare(StatusClass::NotFound), + message: |mapping| capitalized(mapping, ""), + debug: false, + }, + Rule { + when: |mapping| status_is(mapping, 408), + kind: Kind::Timeout(None), + message: |mapping| capitalized(mapping, ""), + debug: false, + }, + Rule { + when: |mapping| status_is(mapping, 429), + kind: stubbed(StatusClass::RateLimit, 429, VERTEX_URL_WITH_SPACE), + message: |mapping| format!("litellm.RateLimitError: {}", capitalized(mapping, "")), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, 500), + kind: Kind::Status { + class: StatusClass::InternalServer, + response: ResponseChoice::InternalServerStub, + }, + message: |mapping| capitalized(mapping, " InternalServerError"), + debug: true, + }, + Rule { + when: |mapping| status_is(mapping, 502), + kind: Kind::ApiConnection, + message: |mapping| capitalized(mapping, ""), + debug: false, + }, + Rule { + when: |mapping| status_is(mapping, 503), + kind: bare(StatusClass::ServiceUnavailable), + message: |mapping| capitalized(mapping, ""), + debug: false, + }, +]; + +pub(super) fn map(mapping: &Mapping<'_>) -> Option { + apply(RULES, mapping).map(|failure| keep_upstream_response(mapping, failure)) +} + +/// Deliberate divergence from `_map_vertex_exception`, which replaces the provider response +/// with a stub and so drops the upstream body and `retry-after`. The response keeps the +/// status the public class carries. +fn keep_upstream_response(mapping: &Mapping<'_>, failure: PublicFailure) -> PublicFailure { + let (PublicKind::Status { status_class, .. }, Some(upstream), false) = ( + &failure.kind, + &mapping.original.response, + mapping.original.status_is_synthesized, + ) else { + return failure; + }; + PublicFailure { + kind: PublicKind::Status { + status_class: *status_class, + response: Some(ResponseArg::Upstream(UpstreamResponse { + status: status_class.status_code(), + ..upstream.clone() + })), + }, + ..failure + } +} + +#[cfg(test)] +mod tests { + use super::super::testing::{context, failure, http, status, upstream, with_debug}; + use super::super::{ExceptionFamily, HttpStub, OriginalException}; + use super::*; + + fn mapped(original: &OriginalException) -> Option { + let context = context("vertex_ai", ExceptionFamily::VertexAi); + map(&Mapping::new(&context, original)) + } + + fn kept(class: StatusClass, body: &str) -> PublicKind { + status(class, upstream(class.status_code(), body)) + } + + #[rstest::rstest] + #[case::api_not_enabled( + 400, + "Vertex AI API has not been used in project x", + with_debug(failure( + kept( + StatusClass::BadRequest, + "Vertex AI API has not been used in project x" + ), + "litellm.BadRequestError: vertex_aiException - Vertex AI API has not been used in project x", + "vertex_ai", + )) + )] + #[case::project_not_found( + 400, + "Unable to find your project", + with_debug(failure( + kept(StatusClass::BadRequest, "Unable to find your project"), + "litellm.BadRequestError: vertex_aiException - Unable to find your project", + "vertex_ai", + )) + )] + #[case::payload_too_large( + 400, + "400 Request payload size exceeds the limit", + failure( + kept( + StatusClass::ContextWindowExceeded, + "400 Request payload size exceeds the limit" + ), + "Vertex_aiException - 400 Request payload size exceeds the limit", + "vertex_ai", + ) + )] + #[case::context_window( + 500, + "This model's maximum context length is 10", + with_debug(failure( + kept( + StatusClass::ContextWindowExceeded, + "This model's maximum context length is 10" + ), + "ContextWindowExceededError: Vertex_aiException - This model's maximum context length is 10", + "vertex_ai", + )) + )] + #[case::unknown_error( + 400, + "None Unknown Error.", + with_debug(failure( + kept(StatusClass::InternalServer, "None Unknown Error."), + "litellm.InternalServerError: vertex_aiException - None Unknown Error.", + "vertex_ai", + )) + )] + #[case::no_parts( + 400, + "Content has no parts.", + with_debug(failure( + kept(StatusClass::InternalServer, "Content has no parts."), + "litellm.InternalServerError: vertex_aiException - Content has no parts.", + "vertex_ai", + )) + )] + #[case::api_key_not_valid( + 400, + "API key not valid.", + with_debug(failure( + kept(StatusClass::Authentication, "API key not valid."), + "Vertex_aiException - API key not valid.", + "vertex_ai", + )) + )] + #[case::forbidden_text( + 400, + "got a 403", + with_debug(failure( + kept(StatusClass::BadRequest, "got a 403"), + "Vertex_aiException BadRequestError - got a 403", + "vertex_ai", + )) + )] + #[case::response_blocked( + 400, + "The response was blocked.", + with_debug(failure( + kept(StatusClass::ContentPolicyViolation, "The response was blocked."), + "Vertex_aiException ContentPolicyViolationError - The response was blocked.", + "vertex_ai", + )) + )] + #[case::output_blocked( + 400, + "Output blocked by content filtering policy", + with_debug(failure( + kept( + StatusClass::ContentPolicyViolation, + "Output blocked by content filtering policy" + ), + "Vertex_aiException ContentPolicyViolationError - Output blocked by content filtering policy", + "vertex_ai", + )) + )] + #[case::quota_marker( + 400, + "Quota exceeded for aiplatform", + with_debug(failure( + kept(StatusClass::RateLimit, "Quota exceeded for aiplatform"), + "litellm.RateLimitError: vertex_aiException - Quota exceeded for aiplatform", + "vertex_ai", + )) + )] + #[case::wrapped_429( + 503, + r#"{"error": {"code": "429"}}"#, + with_debug(failure( + kept(StatusClass::RateLimit, r#"{"error": {"code": "429"}}"#), + r#"litellm.RateLimitError: vertex_aiException - {"error": {"code": "429"}}"#, + "vertex_ai", + )) + )] + #[case::overloaded( + 400, + "The model is overloaded.", + with_debug(failure( + kept(StatusClass::InternalServer, "The model is overloaded."), + "litellm.InternalServerError: vertex_aiException - The model is overloaded.", + "vertex_ai", + )) + )] + #[case::internal_server_text( + 400, + "500 Internal Server Error", + with_debug(failure( + kept(StatusClass::InternalServer, "500 Internal Server Error"), + "litellm.InternalServerError: vertex_aiException - 500 Internal Server Error", + "vertex_ai", + )) + )] + fn each_text_rule_maps_by_the_body( + #[case] status_code: u16, + #[case] body: &str, + #[case] expected: PublicFailure, + ) { + assert_eq!(mapped(&http(status_code, body)), Some(expected)); + } + + #[rstest::rstest] + #[case::bad_request( + 400, + with_debug(failure( + kept(StatusClass::BadRequest, "rejected"), + "Vertex_aiException BadRequestError - rejected", + "vertex_ai" + )) + )] + #[case::authentication( + 401, + failure( + kept(StatusClass::Authentication, "rejected"), + "Vertex_aiException - rejected", + "vertex_ai" + ) + )] + #[case::permission_denied( + 403, + failure( + kept(StatusClass::PermissionDenied, "rejected"), + "Vertex_aiException - rejected", + "vertex_ai" + ) + )] + #[case::not_found( + 404, + failure( + kept(StatusClass::NotFound, "rejected"), + "Vertex_aiException - rejected", + "vertex_ai" + ) + )] + #[case::request_timeout(408, failure(PublicKind::Timeout { status: None }, "Vertex_aiException - rejected", "vertex_ai"))] + #[case::rate_limited( + 429, + with_debug(failure( + kept(StatusClass::RateLimit, "rejected"), + "litellm.RateLimitError: Vertex_aiException - rejected", + "vertex_ai" + )) + )] + #[case::internal_server( + 500, + with_debug(failure( + kept(StatusClass::InternalServer, "rejected"), + "Vertex_aiException InternalServerError - rejected", + "vertex_ai" + )) + )] + #[case::bad_gateway( + 502, + failure( + PublicKind::ApiConnection, + "Vertex_aiException - rejected", + "vertex_ai" + ) + )] + #[case::service_unavailable( + 503, + failure( + kept(StatusClass::ServiceUnavailable, "rejected"), + "Vertex_aiException - rejected", + "vertex_ai" + ) + )] + fn each_status_rule_maps_by_the_status( + #[case] status_code: u16, + #[case] expected: PublicFailure, + ) { + assert_eq!(mapped(&http(status_code, "rejected")), Some(expected)); + } + + #[rstest::rstest] + #[case::unmapped_status(409)] + #[case::gateway_timeout(504)] + fn statuses_without_a_rule_fall_through(#[case] status_code: u16) { + assert_eq!(mapped(&http(status_code, "rejected")), None); + } + + #[rstest::rstest] + #[case::stub_without_an_upstream_response( + OriginalException::Response { message: "got a 403".into() }, + status(StatusClass::BadRequest, Some(ResponseArg::Stub(HttpStub { status: 403, method: "POST", url: VERTEX_URL_WITH_SPACE, content: None }))) + )] + #[case::stub_for_a_synthesized_status( + OriginalException::Connection { message: "got a 403".into() }, + status(StatusClass::BadRequest, Some(ResponseArg::Stub(HttpStub { status: 403, method: "POST", url: VERTEX_URL_WITH_SPACE, content: None }))) + )] + fn the_rule_response_stays_when_there_is_no_real_upstream_response( + #[case] original: OriginalException, + #[case] kind: PublicKind, + ) { + assert_eq!(mapped(&original).map(|failure| failure.kind), Some(kind)); + } + + #[test] + fn a_synthesized_500_keeps_the_internal_server_stub() { + let original = OriginalException::Connection { + message: "refused".into(), + }; + assert_eq!( + mapped(&original), + Some(with_debug(failure( + status( + StatusClass::InternalServer, + Some(ResponseArg::Stub(HttpStub { + status: 500, + method: "completion", + url: "https://github.com/BerriAI/litellm", + content: Some("refused".into()), + })) + ), + "Vertex_aiException InternalServerError - refused", + "vertex_ai" + ))) + ); + } + + #[rstest::rstest] + #[case::project_before_payload_size( + "Unable to find your project 400 Request payload size exceeds", + StatusClass::BadRequest, + "litellm.BadRequestError: vertex_aiException - Unable to find your project 400 Request payload size exceeds", + true + )] + #[case::payload_size_before_context_window( + "400 Request payload size exceeds; This model's maximum context length is 10", + StatusClass::ContextWindowExceeded, + "Vertex_aiException - 400 Request payload size exceeds; This model's maximum context length is 10", + false + )] + #[case::api_key_before_forbidden( + "API key not valid. 403", + StatusClass::Authentication, + "Vertex_aiException - API key not valid. 403", + true + )] + #[case::forbidden_before_blocked( + "403 The response was blocked.", + StatusClass::BadRequest, + "Vertex_aiException BadRequestError - 403 The response was blocked.", + true + )] + #[case::blocked_before_quota( + "The response was blocked. Resource exhausted", + StatusClass::ContentPolicyViolation, + "Vertex_aiException ContentPolicyViolationError - The response was blocked. Resource exhausted", + true + )] + #[case::quota_before_overloaded( + "Resource exhausted The model is overloaded.", + StatusClass::RateLimit, + "litellm.RateLimitError: vertex_aiException - Resource exhausted The model is overloaded.", + true + )] + fn the_earlier_rule_wins_when_two_apply( + #[case] body: &str, + #[case] class: StatusClass, + #[case] message: &str, + #[case] debug: bool, + ) { + let expected = failure(kept(class, body), message, "vertex_ai"); + assert_eq!( + mapped(&http(401, body)), + Some(if debug { + with_debug(expected) + } else { + expected + }) + ); + } +} diff --git a/litellm-rust/crates/core-utils/src/lib.rs b/litellm-rust/crates/core-utils/src/lib.rs index a8895cccf2c..0c26aa50cc3 100644 --- a/litellm-rust/crates/core-utils/src/lib.rs +++ b/litellm-rust/crates/core-utils/src/lib.rs @@ -1,7 +1,10 @@ pub mod call_arguments; pub mod core_helpers; +pub mod exception_mapping_utils; pub mod get_llm_provider_logic; pub mod params; pub mod prompt_templates; +pub mod python_repr; +pub mod secret_redaction; pub mod serde_compat; pub mod url_utils; diff --git a/litellm-rust/crates/core-utils/src/python_repr.rs b/litellm-rust/crates/core-utils/src/python_repr.rs new file mode 100644 index 00000000000..7ccfb826377 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/python_repr.rs @@ -0,0 +1,93 @@ +/// `repr()` of a Python `str`: single quotes unless the text holds a single quote and no +/// double quote, with backslashes, the chosen quote and control characters escaped. +pub fn python_str_repr(value: &str) -> String { + let quote = if value.contains('\'') && !value.contains('"') { + '"' + } else { + '\'' + }; + let escaped: String = value + .chars() + .map(|character| match character { + '\\' => "\\\\".to_string(), + '\t' => "\\t".to_string(), + '\n' => "\\n".to_string(), + '\r' => "\\r".to_string(), + character if character == quote => format!("\\{character}"), + character + if (character as u32) < 0x20 || (0x7f..0xa0).contains(&(character as u32)) => + { + format!("\\x{:02x}", character as u32) + } + character => character.to_string(), + }) + .collect(); + format!("{quote}{escaped}{quote}") +} + +/// `repr()` of the Python value a JSON value decodes to. +pub fn python_value_repr(value: &serde_json::Value) -> String { + use serde_json::Value; + match value { + Value::Null => "None".to_string(), + Value::Bool(true) => "True".to_string(), + Value::Bool(false) => "False".to_string(), + Value::Number(number) => number.to_string(), + Value::String(text) => python_str_repr(text), + Value::Array(items) => format!( + "[{}]", + items + .iter() + .map(python_value_repr) + .collect::>() + .join(", ") + ), + Value::Object(fields) => format!( + "{{{}}}", + fields + .iter() + .map(|(key, value)| format!( + "{}: {}", + python_str_repr(key), + python_value_repr(value) + )) + .collect::>() + .join(", ") + ), + } +} + +#[cfg(test)] +mod tests { + use serde_json::json; + + use super::{python_str_repr, python_value_repr}; + + #[rstest::rstest] + #[case::null(json!(null), "None")] + #[case::true_(json!(true), "True")] + #[case::false_(json!(false), "False")] + #[case::integer(json!(5), "5")] + #[case::float(json!(1.5), "1.5")] + #[case::string(json!("it's"), "\"it's\"")] + #[case::list(json!(["a", 1]), "['a', 1]")] + #[case::dict(json!({"format": "native"}), "{'format': 'native'}")] + #[case::empty_list(json!([]), "[]")] + fn value_repr_matches_python(#[case] value: serde_json::Value, #[case] expected: &str) { + assert_eq!(python_value_repr(&value), expected); + } + + #[rstest::rstest] + #[case::plain("native", "'native'")] + #[case::single_quote("it's", "\"it's\"")] + #[case::both_quotes("it's \"x\"", "'it\\'s \"x\"'")] + #[case::double_quote("say \"x\"", "'say \"x\"'")] + #[case::backslash("a\\b", "'a\\\\b'")] + #[case::whitespace("a\tb\nc\rd", "'a\\tb\\nc\\rd'")] + #[case::control("a\u{1}b\u{7f}c\u{85}", "'a\\x01b\\x7fc\\x85'")] + #[case::unicode("café", "'café'")] + #[case::empty("", "''")] + fn matches_python_repr(#[case] value: &str, #[case] expected: &str) { + assert_eq!(python_str_repr(value), expected); + } +} diff --git a/litellm-rust/crates/core-utils/src/secret_redaction.rs b/litellm-rust/crates/core-utils/src/secret_redaction.rs new file mode 100644 index 00000000000..32922d7430c --- /dev/null +++ b/litellm-rust/crates/core-utils/src/secret_redaction.rs @@ -0,0 +1,97 @@ +use std::sync::LazyLock; + +use fancy_regex::Regex; + +pub const REDACTED: &str = "REDACTED"; + +const DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH: usize = 16; + +fn minimum_custom_key_length() -> usize { + std::env::var("MINIMUM_CUSTOM_KEY_LENGTH") + .ok() + .and_then(|value| value.trim().parse().ok()) + .unwrap_or(DEFAULT_MINIMUM_CUSTOM_KEY_LENGTH) +} + +fn secret_patterns(minimum_custom_key_length: usize) -> String { + let sk_suffix_length = minimum_custom_key_length.saturating_sub("sk-".len()); + [ + r"-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----", + r"\bya29\.[A-Za-z0-9_.~+/-]+", + r#"(?:client_secret|azure_password|azure_username)\s+[^\s,'"})\]{}>]+"#, + r"(?:AKIA|ASIA)[0-9A-Z]{16}", + r"Bearer\s+[A-Za-z0-9\-._~+/]{10,}=*", + r"Basic\s+[A-Za-z0-9+/]{10,}={0,2}", + &format!(r"sk-[A-Za-z0-9\-_]{{{sk_suffix_length},}}"), + r#"(?<=[?&])(?:api[_-]?key|\w*(?:token|password|passwd|client_secret|secret_key|_secret))=[^\s&'"]+"#, + r#"(?:api[_-]?key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]{8,}"#, + r#"(?:x-api-key|api-key)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, + r"x-ak-[A-Za-z0-9\-_]{20,}", + r"AIza[0-9A-Za-z\-_]{35}", + r#"(?<=[?&])key=[^\s&'"]{8,}"#, + r#"(?:^|(?<=\W))\w*(?:password|passwd|client_secret|secret_key|_secret)['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, + r#"(?<=://)[^\s'":]{0,4096}:[^\s'"]{1,4096}(?=@)"#, + r"dapi[0-9a-f]{32}", + r#"litellm\.[A-Za-z0-9_]*_key['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, + r#"private_key['"]?\s*[:=]\s*['"]?(?:-----BEGIN[A-Z \-]*PRIVATE KEY-----[\s\S]*?-----END[A-Z \-]*PRIVATE KEY-----|[^\s,'"})\]{}>]+)"#, + concat!( + r"(?:master_key|xai_key|database_url|db_url|connection_string|", + r"aws_secret_access_key|aws_session_token|aws_access_key_id|", + r"signing_key|encryption_key|", + r"auth_token|access_token|refresh_token|", + r"slack_webhook_url|webhook_url|", + r"database_connection_string|", + r"huggingface_token|jwt_secret)", + r#"['"]?\s*[:=]\s*['"]?[^\s,'"})\]{}>]+"#, + ), + r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*", + r"(?<=[?&])sig=[A-Za-z0-9%+/=]+", + r#"\{[^{}]*"type"\s*:\s*"service_account"[^{}]*(?:\{[^{}]*\}[^{}]*)*\}"#, + ] + .join("|") +} + +static SECRET_RE: LazyLock = LazyLock::new(|| { + Regex::new(&format!( + "(?i){}", + secret_patterns(minimum_custom_key_length()) + )) + .expect("secret redaction patterns compile") +}); + +pub fn redact_string(value: &str) -> String { + SECRET_RE.replace_all(value, REDACTED).into_owned() +} + +pub fn secret_redaction_enabled() -> bool { + !std::env::var("LITELLM_DISABLE_REDACT_SECRETS") + .is_ok_and(|value| value.eq_ignore_ascii_case("true")) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[rstest::rstest] + #[case::bearer("auth failed: Bearer abcdefghijklmnop", "auth failed: REDACTED")] + #[case::sk_key("key sk-abcdefghijklmnopqrstuvwxyz rejected", "key REDACTED rejected")] + #[case::short_sk_key_is_kept("sk-abc", "sk-abc")] + #[case::query_param("GET /v1?api_key=secret123&x=1", "GET /v1?REDACTED&x=1")] + #[case::dict_repr("{'api_key': 'abcdefghij'}", "{'REDACTED'}")] + #[case::url_credentials("postgres://user:pass@host/db", "postgres://REDACTED@host/db")] + #[case::case_insensitive("BEARER ABCDEFGHIJKLMNOP", "REDACTED")] + #[case::aws_key("AKIAABCDEFGHIJKLMNOP", "REDACTED")] + #[case::sas_signature("https://x.blob/a?sv=1&sig=abc%2B=", "https://x.blob/a?sv=1&REDACTED")] + #[case::password_needs_word_boundary("db_password=hunter2", "REDACTED")] + #[case::plain_text_is_kept(r#"{"message": "rejected"}"#, r#"{"message": "rejected"}"#)] + fn redacts_the_same_spans_as_the_python_patterns(#[case] input: &str, #[case] expected: &str) { + assert_eq!(redact_string(input), expected); + } + + #[test] + fn sk_threshold_follows_the_minimum_custom_key_length() { + let patterns = Regex::new(&format!("(?i){}", secret_patterns(8))).unwrap(); + assert_eq!(patterns.replace_all("sk-abcde", REDACTED), REDACTED); + assert_eq!(patterns.replace_all("sk-abcd", REDACTED), "sk-abcd"); + } +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/api.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/api.json new file mode 100644 index 00000000000..ae629114589 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/api.json @@ -0,0 +1,13 @@ +{ + "kind": { + "type": "api", + "status": 409, + "request_url": "https://docs.litellm.ai/docs" + }, + "message": "MistralException - api", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": null, + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/api_connection.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/api_connection.json new file mode 100644 index 00000000000..ab400ed9e01 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/api_connection.json @@ -0,0 +1,11 @@ +{ + "kind": { + "type": "api_connection" + }, + "message": "MistralException - api_connection", + "model": "ocr-model", + "llm_provider": null, + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": null, + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_authentication.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_authentication.json new file mode 100644 index 00000000000..057392575a1 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_authentication.json @@ -0,0 +1,13 @@ +{ + "kind": { + "type": "status", + "status_class": "authentication", + "response": null + }, + "message": "MistralException - status_authentication", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": null, + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_bad_gateway.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_bad_gateway.json new file mode 100644 index 00000000000..abb1425f686 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_bad_gateway.json @@ -0,0 +1,28 @@ +{ + "kind": { + "type": "status", + "status_class": "bad_gateway", + "response": { + "type": "upstream", + "status": 502, + "body": "{\"message\": \"rejected\"}", + "headers": [ + [ + "retry-after", + "7" + ] + ] + } + }, + "message": "MistralException - status_bad_gateway", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": [ + [ + "retry-after", + "7" + ] + ], + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_bad_request.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_bad_request.json new file mode 100644 index 00000000000..171a994cd35 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_bad_request.json @@ -0,0 +1,28 @@ +{ + "kind": { + "type": "status", + "status_class": "bad_request", + "response": { + "type": "upstream", + "status": 400, + "body": "{\"message\": \"rejected\"}", + "headers": [ + [ + "retry-after", + "7" + ] + ] + } + }, + "message": "MistralException - status_bad_request", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": [ + [ + "retry-after", + "7" + ] + ], + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_content_policy_violation.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_content_policy_violation.json new file mode 100644 index 00000000000..ae9c1e145de --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_content_policy_violation.json @@ -0,0 +1,28 @@ +{ + "kind": { + "type": "status", + "status_class": "content_policy_violation", + "response": { + "type": "upstream", + "status": 400, + "body": "{\"message\": \"rejected\"}", + "headers": [ + [ + "retry-after", + "7" + ] + ] + } + }, + "message": "MistralException - status_content_policy_violation", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": [ + [ + "retry-after", + "7" + ] + ], + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_context_window_exceeded.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_context_window_exceeded.json new file mode 100644 index 00000000000..61e1a56a622 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_context_window_exceeded.json @@ -0,0 +1,28 @@ +{ + "kind": { + "type": "status", + "status_class": "context_window_exceeded", + "response": { + "type": "upstream", + "status": 400, + "body": "{\"message\": \"rejected\"}", + "headers": [ + [ + "retry-after", + "7" + ] + ] + } + }, + "message": "MistralException - status_context_window_exceeded", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": [ + [ + "retry-after", + "7" + ] + ], + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_internal_server.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_internal_server.json new file mode 100644 index 00000000000..b3c5c51a785 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_internal_server.json @@ -0,0 +1,19 @@ +{ + "kind": { + "type": "status", + "status_class": "internal_server", + "response": { + "type": "stub", + "status": 500, + "method": "completion", + "url": "https://github.com/BerriAI/litellm", + "content": "upstream text" + } + }, + "message": "MistralException - status_internal_server", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": null, + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_not_found.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_not_found.json new file mode 100644 index 00000000000..26ffd872961 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_not_found.json @@ -0,0 +1,28 @@ +{ + "kind": { + "type": "status", + "status_class": "not_found", + "response": { + "type": "upstream", + "status": 404, + "body": "{\"message\": \"rejected\"}", + "headers": [ + [ + "retry-after", + "7" + ] + ] + } + }, + "message": "MistralException - status_not_found", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": [ + [ + "retry-after", + "7" + ] + ], + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_permission_denied.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_permission_denied.json new file mode 100644 index 00000000000..42772f98830 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_permission_denied.json @@ -0,0 +1,19 @@ +{ + "kind": { + "type": "status", + "status_class": "permission_denied", + "response": { + "type": "stub", + "status": 403, + "method": "POST", + "url": " https://cloud.google.com/vertex-ai/", + "content": null + } + }, + "message": "MistralException - status_permission_denied", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": null, + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_rate_limit.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_rate_limit.json new file mode 100644 index 00000000000..c9b88822ebd --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_rate_limit.json @@ -0,0 +1,28 @@ +{ + "kind": { + "type": "status", + "status_class": "rate_limit", + "response": { + "type": "upstream", + "status": 429, + "body": "{\"message\": \"rejected\"}", + "headers": [ + [ + "retry-after", + "7" + ] + ] + } + }, + "message": "MistralException - status_rate_limit", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": [ + [ + "retry-after", + "7" + ] + ], + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_service_unavailable.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_service_unavailable.json new file mode 100644 index 00000000000..5af65202ef1 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_service_unavailable.json @@ -0,0 +1,28 @@ +{ + "kind": { + "type": "status", + "status_class": "service_unavailable", + "response": { + "type": "upstream", + "status": 503, + "body": "{\"message\": \"rejected\"}", + "headers": [ + [ + "retry-after", + "7" + ] + ] + } + }, + "message": "MistralException - status_service_unavailable", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": [ + [ + "retry-after", + "7" + ] + ], + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/status_unsupported_params.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_unsupported_params.json new file mode 100644 index 00000000000..a1b318ce03c --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/status_unsupported_params.json @@ -0,0 +1,28 @@ +{ + "kind": { + "type": "status", + "status_class": "unsupported_params", + "response": { + "type": "upstream", + "status": 400, + "body": "{\"message\": \"rejected\"}", + "headers": [ + [ + "retry-after", + "7" + ] + ] + } + }, + "message": "MistralException - status_unsupported_params", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": [ + [ + "retry-after", + "7" + ] + ], + "print_banner": false +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/timeout_with_status.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/timeout_with_status.json new file mode 100644 index 00000000000..8b215bdb7e6 --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/timeout_with_status.json @@ -0,0 +1,12 @@ +{ + "kind": { + "type": "timeout", + "status": 504 + }, + "message": "MistralException - timeout_with_status", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": "\nModel: ocr-model", + "litellm_response_headers": null, + "print_banner": true +} diff --git a/tests/test_litellm/rust_bridge/fixtures/public_failures/timeout_without_status.json b/tests/test_litellm/rust_bridge/fixtures/public_failures/timeout_without_status.json new file mode 100644 index 00000000000..4a79169776e --- /dev/null +++ b/tests/test_litellm/rust_bridge/fixtures/public_failures/timeout_without_status.json @@ -0,0 +1,12 @@ +{ + "kind": { + "type": "timeout", + "status": null + }, + "message": "MistralException - timeout_without_status", + "model": "ocr-model", + "llm_provider": "mistral", + "litellm_debug_info": null, + "litellm_response_headers": null, + "print_banner": false +}