diff --git a/litellm-rust/crates/core-utils/src/decline.rs b/litellm-rust/crates/core-utils/src/decline.rs new file mode 100644 index 00000000000..6f398b16a06 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/decline.rs @@ -0,0 +1,16 @@ +/// Rust core cannot serve this request, and nothing was sent to the provider yet. +/// +/// Every error on the call path carries it unchanged, so the host can hand the same request +/// to another implementation: the SDK runs legacy Python, the gateway forwards to the proxy. +/// Raise it only before the provider call. Anything after the send is terminal. +#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)] +#[error("declined by the rust path: {reason}")] +pub struct Decline { + pub reason: &'static str, +} + +impl Decline { + pub const fn new(reason: &'static str) -> Self { + Self { reason } + } +} diff --git a/litellm-rust/crates/core-utils/src/lib.rs b/litellm-rust/crates/core-utils/src/lib.rs index a937f55654e..cfbf451bf1b 100644 --- a/litellm-rust/crates/core-utils/src/lib.rs +++ b/litellm-rust/crates/core-utils/src/lib.rs @@ -1,5 +1,6 @@ pub mod call_arguments; pub mod core_helpers; +pub mod decline; pub mod dot_notation_indexing; pub mod exception_mapping_utils; pub mod get_llm_provider_logic; diff --git a/litellm-rust/crates/gateway-inference/src/error.rs b/litellm-rust/crates/gateway-inference/src/error.rs index ec82a6ede12..e1470061190 100644 --- a/litellm-rust/crates/gateway-inference/src/error.rs +++ b/litellm-rust/crates/gateway-inference/src/error.rs @@ -49,9 +49,8 @@ impl Error { pub fn status(&self) -> StatusCode { match self { Self::Auth(error) => error.status(), - Self::Unsupported(_) - | Self::Route(InferenceError::Unsupported(_)) - | Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED, + Self::Unsupported(_) => StatusCode::NOT_IMPLEMENTED, + Self::Route(error) if error.declined().is_some() => StatusCode::NOT_IMPLEMENTED, Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE, Self::Ocr( OcrError::Auth(litellm_auth::Error::MissingApiKey { .. }) diff --git a/litellm-rust/crates/inference-chat/src/common_utils.rs b/litellm-rust/crates/inference-chat/src/common_utils.rs index 4063f33432d..c4021d80f07 100644 --- a/litellm-rust/crates/inference-chat/src/common_utils.rs +++ b/litellm-rust/crates/inference-chat/src/common_utils.rs @@ -61,7 +61,7 @@ pub(super) fn resolve_request( )); } if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) { - return Err(Error::Unsupported(reason.0)); + return Err(Error::Declined(reason)); } Ok(ResolvedChatCompletionsRequest { messages, diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index 595e851c6fd..1857d066cae 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -2,6 +2,7 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender}; use std::time::Duration; use litellm_auth::AuthServices; +use litellm_core_utils::decline::Decline; use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest}; use litellm_http::{ Client, @@ -145,29 +146,12 @@ pub(super) async fn execute( config .transform_response(provider.model, ProviderChatResponseData { body }) .map_err(Error::from) - .map_err(as_response_error) + .map_err(Error::after_send) }, ) .await } -/// Re-tag an error raised while normalizing a response the provider already -/// returned. -/// -/// A config reports the same variants on either side of the call: a missing -/// field or an unsupported block can mean "this request cannot be translated" -/// before the request is sent and "this response cannot be normalized" here. Only the -/// second kind has already been billed, and a host that keeps a reference -/// implementation must not retry those, so collapse them to one variant that -/// can only mean the provider was already called. -pub(super) fn as_response_error(err: Error) -> Error { - match err { - already @ (Error::InvalidResponse(_) - | Error::Transport(litellm_http::transport::Error::Http { .. })) => already, - other => Error::InvalidResponse(other.to_string().into()), - } -} - pub(super) fn outbound_request( authenticated: Authenticated, url: String, @@ -184,7 +168,7 @@ pub(super) fn outbound_request( // Python drops the caller's copy and prefers a forwarded Authorization // over the signature, so leave the request to it. litellm_http::Error::ComputedHeader(_) => { - Error::Unsupported("request forwards a header AWS SigV4 computes") + Error::Declined(Decline::new("request forwards a header AWS SigV4 computes")) } other => Error::Http(other), }) @@ -348,25 +332,4 @@ mod tests { )); assert!(interceptors.raw.into_inner().unwrap().is_empty()); } - - #[rstest::rstest] - fn response_errors_collapse_to_one_variant_that_can_only_mean_already_sent() { - for original in [ - Error::MissingField("usage"), - Error::Unsupported("non-text response content block"), - Error::InvalidRequest("whatever".to_string().into()), - Error::Auth(litellm_auth::Error::InvalidHeader), - ] { - let label = format!("{original:?}"); - assert!( - matches!(as_response_error(original), Error::InvalidResponse(_)), - "{label} must not stay retryable once the provider has answered" - ); - } - let upstream = Error::Transport(litellm_http::transport::Error::Http { - status: 500, - body: "boom".to_string(), - }); - assert_eq!(as_response_error(upstream.clone()), upstream); - } } diff --git a/litellm-rust/crates/inference-chat/tests/request.rs b/litellm-rust/crates/inference-chat/tests/request.rs index 5988a7baee5..331cb122007 100644 --- a/litellm-rust/crates/inference-chat/tests/request.rs +++ b/litellm-rust/crates/inference-chat/tests/request.rs @@ -355,7 +355,7 @@ async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str) let error = complete(call) .await .expect_err("conflicting signing headers fail"); - assert!(matches!(error, Error::Unsupported(_)), "{error:?}"); + assert!(error.declined().is_some(), "{error:?}"); assert!(received(&upstream).await.is_empty()); } diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 55c80d9bddf..f554d3b7223 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -325,6 +325,7 @@ fn decode_response( config .transform_anthropic_messages_response(model, response) .map_err(Error::from) + .map_err(Error::after_send) } fn streaming_response( diff --git a/litellm-rust/crates/inference-messages/tests/messages/host.rs b/litellm-rust/crates/inference-messages/tests/messages/host.rs index ba20fb9eec4..9bf7ab36b35 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/host.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/host.rs @@ -81,7 +81,7 @@ impl litellm_host::interceptors::Interceptors< Result<(), Error> { self.facts.lock().unwrap().push(facts); if self.reject_result { - return Err(Error::Unsupported("result rejected")); + return Err(Error::InvalidRequest("result rejected".into())); } Ok(()) } @@ -181,7 +181,7 @@ async fn rejected_results_are_not_delivered_or_cached( assert_eq!( result, if reject { - Err(Error::Unsupported("result rejected")) + Err(Error::InvalidRequest("result rejected".into())) } else { Ok(()) } diff --git a/litellm-rust/crates/inference-responses/src/handler.rs b/litellm-rust/crates/inference-responses/src/handler.rs index 02182d23ae3..806df57faba 100644 --- a/litellm-rust/crates/inference-responses/src/handler.rs +++ b/litellm-rust/crates/inference-responses/src/handler.rs @@ -129,6 +129,7 @@ pub(super) async fn execute( .transform_response_api_response(value) .map(ResponsesOutput::Complete) .map_err(Error::from) + .map_err(Error::after_send) }, ) .await diff --git a/litellm-rust/crates/inference-responses/src/lib.rs b/litellm-rust/crates/inference-responses/src/lib.rs index b6e151ecec3..9af5dea1fe8 100644 --- a/litellm-rust/crates/inference-responses/src/lib.rs +++ b/litellm-rust/crates/inference-responses/src/lib.rs @@ -11,6 +11,7 @@ mod handler; use std::sync::Arc; use litellm_auth::AuthServices; +use litellm_core_utils::decline::Decline; use litellm_core_utils::get_llm_provider_logic::{ CustomLlmProvider, LlmProviders, get_custom_llm_provider, }; @@ -24,6 +25,8 @@ use litellm_llms::{ use litellm_secrets::source::SecretSource; use types::{ResponsesCall, ResponsesOutput, ResponsesProvider}; +const NOT_NATIVE: Decline = Decline::new("native HTTP responses provider"); + fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> { match provider { LlmProviders::Openai => Some(&OpenAiResponsesApiConfig), @@ -111,9 +114,8 @@ impl ResponsesRoute { } let provider = provider_name .parse::() - .map_err(|_| Error::Unsupported("native HTTP responses provider"))?; - let config = responses_provider(provider) - .ok_or(Error::Unsupported("native HTTP responses provider"))?; + .map_err(|_| Error::Declined(NOT_NATIVE))?; + let config = responses_provider(provider).ok_or(Error::Declined(NOT_NATIVE))?; let wire_model = match provider { LlmProviders::Openai if resolved_model.contains('/') => { return Err(Error::InvalidProvider(call.model)); @@ -123,7 +125,7 @@ impl ResponsesRoute { format!("openrouter/{resolved_model}") } LlmProviders::Openrouter => resolved_model.into(), - _ => return Err(Error::Unsupported("native HTTP responses provider")), + _ => return Err(Error::Declined(NOT_NATIVE)), }; let provider = ResponsesProvider { identity: ProviderIdentity { diff --git a/litellm-rust/crates/inference-responses/tests/responses.rs b/litellm-rust/crates/inference-responses/tests/responses.rs index ae07a71dabd..38f44dae299 100644 --- a/litellm-rust/crates/inference-responses/tests/responses.rs +++ b/litellm-rust/crates/inference-responses/tests/responses.rs @@ -103,10 +103,7 @@ async fn unsupported_providers_fail_before_transport( ) .await; - assert!(matches!( - result, - Err(Error::Unsupported("native HTTP responses provider")) - )); + assert!(matches!(result, Err(ref error) if error.declined().is_some())); assert!(received(&upstream).await.is_empty()); } diff --git a/litellm-rust/crates/inference-transcription/src/handler.rs b/litellm-rust/crates/inference-transcription/src/handler.rs index 5a7238c5e36..1793300a5cd 100644 --- a/litellm-rust/crates/inference-transcription/src/handler.rs +++ b/litellm-rust/crates/inference-transcription/src/handler.rs @@ -76,7 +76,8 @@ pub(super) async fn execute( error, )) })?; - Ok(config - .transform_audio_transcription_response(provider.model, response_json)? - .into_json()) + config + .transform_audio_transcription_response(provider.model, response_json) + .map(|response| response.into_json()) + .map_err(|error| Error::from(error).after_send()) } diff --git a/litellm-rust/crates/inference/src/error.rs b/litellm-rust/crates/inference/src/error.rs index 895c8c55e01..5dc92461138 100644 --- a/litellm-rust/crates/inference/src/error.rs +++ b/litellm-rust/crates/inference/src/error.rs @@ -7,6 +7,7 @@ use std::sync::Arc; +use litellm_core_utils::decline::Decline; use litellm_http::transport::Error as TransportError; use litellm_llms::Error as LlmError; @@ -25,8 +26,8 @@ pub enum Error { InvalidRequest(#[source] litellm_llms::ErrorDetail), #[error("invalid response: {0}")] InvalidResponse(#[source] litellm_llms::ErrorDetail), - #[error("unsupported by the rust path: {0}")] - Unsupported(&'static str), + #[error(transparent)] + Declined(Decline), #[error(transparent)] Auth(#[from] litellm_auth::Error), #[error(transparent)] @@ -64,16 +65,50 @@ impl Error { | Self::MissingField(_) | Self::InvalidProvider(_) | Self::InvalidRequest(_) - | Self::Unsupported(_) | Self::Headers(_) => true, Self::Auth(error) => !matches!(error, litellm_auth::Error::MissingApiKey { .. }), - Self::InvalidResponse(_) + Self::Declined(_) + | Self::InvalidResponse(_) | Self::Transport(_) | Self::Http(_) | Self::Secret(_) | Self::PostCallHook(_) => false, } } + + /// The decline this error carries, if the request never reached the provider. + pub fn declined(&self) -> Option<&Decline> { + match self { + Self::Declined(decline) => Some(decline), + Self::InvalidType { .. } + | Self::MissingField(_) + | Self::InvalidProvider(_) + | Self::InvalidRequest(_) + | Self::InvalidResponse(_) + | Self::Auth(_) + | Self::Transport(_) + | Self::Headers(_) + | Self::Http(_) + | Self::Secret(_) + | Self::PostCallHook(_) => None, + } + } + + /// Re-tag an error raised while normalizing a response the provider already returned. + /// + /// A config reports the same variants on either side of the call: a missing field can + /// mean "this request cannot be translated" before the send and "this response cannot be + /// normalized" after it. Only the second kind was billed, and a host that keeps another + /// implementation must not retry it, so collapse it to the one variant that can only mean + /// the provider was already called. + pub fn after_send(self) -> Self { + match self { + already @ (Self::InvalidResponse(_) | Self::Transport(TransportError::Http { .. })) => { + already + } + other => Self::InvalidResponse(other.to_string().into()), + } + } } impl From for Error { @@ -83,7 +118,7 @@ impl From for Error { LlmError::MissingField(field) => Self::MissingField(field), LlmError::InvalidRequest(message) => Self::InvalidRequest(message), LlmError::InvalidResponse(message) => Self::InvalidResponse(message), - LlmError::Unsupported(reason) => Self::Unsupported(reason), + LlmError::Declined(decline) => Self::Declined(decline), LlmError::Auth(error) => Self::Auth(error), } } @@ -116,9 +151,45 @@ impl Eq for SecretError {} #[cfg(test)] mod tests { use super::Error; + use litellm_core_utils::decline::Decline; use litellm_llms::{Error as LlmError, ErrorDetail}; use rstest::rstest; + #[test] + fn a_transformation_decline_survives_into_the_route_error() { + let decline = Decline::new("unrecognized request parameter"); + assert_eq!( + Error::from(LlmError::Declined(decline)).declined(), + Some(&decline) + ); + } + + #[rstest] + #[case::declined(Error::Declined(Decline::new("non-text response content block")))] + #[case::missing_field(Error::MissingField("usage"))] + #[case::invalid_request(Error::InvalidRequest("whatever".into()))] + #[case::auth(Error::Auth(litellm_auth::Error::InvalidHeader))] + fn errors_after_the_send_are_never_a_decline(#[case] error: Error) { + let collapsed = error.after_send(); + assert!(matches!(collapsed, Error::InvalidResponse(_))); + assert_eq!(collapsed.declined(), None); + } + + #[test] + fn an_upstream_status_stays_as_is_after_the_send() { + let upstream = Error::Transport(litellm_http::transport::Error::Http { + status: 500, + body: "boom".into(), + }); + assert_eq!(upstream.clone().after_send(), upstream); + } + + #[test] + fn a_post_call_hook_failure_is_never_a_decline() { + let hook = Error::post_call(Error::Declined(Decline::new("hook"))); + assert_eq!(hook.declined(), None); + } + #[test] fn a_missing_api_key_is_the_environment_not_the_request() { assert!( diff --git a/litellm-rust/crates/inference/tests/caching.rs b/litellm-rust/crates/inference/tests/caching.rs index a0a6a757565..4566114e981 100644 --- a/litellm-rust/crates/inference/tests/caching.rs +++ b/litellm-rust/crates/inference/tests/caching.rs @@ -204,7 +204,8 @@ async fn streamed( .chunks(3) .map(|bytes| Ok(Bytes::copy_from_slice(bytes))) .collect::>(); - let ending = fail.then_some(Err(Error::Unsupported("test transport failure"))); + let ending = + fail.then_some(Err(Error::InvalidRequest("test transport failure".into()))); Ok(CallOutput::Stream { head: (), chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(), @@ -315,7 +316,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc for CacheHitAccounting { ); self.key.set(key).unwrap(); if self.reject { - return Err(Error::Unsupported("cache accounting rejected")); + return Err(Error::InvalidRequest("cache accounting rejected".into())); } Ok(()) } @@ -569,10 +570,10 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure( .await }; if reject { - assert!(matches!( - result, - Err(Error::Unsupported("cache accounting rejected")) - )); + assert_eq!( + result.unwrap_err(), + Error::InvalidRequest("cache accounting rejected".into()) + ); } else { assert_eq!(result.unwrap(), expected); } diff --git a/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs index b8ea701af99..8c998a362a7 100644 --- a/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/batches/transformation.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::decline::Decline; use litellm_llms_types::formats::batches::{BatchRequestCounts, BatchResponse, BatchStatus}; use litellm_llms_types::formats::messages::MessagesResponse; use litellm_llms_types::providers::anthropic::{BATCHES_PATH, MESSAGES_PATH}; @@ -127,7 +128,9 @@ impl AnthropicBatchesConfig for AnthropicBatchesTransformation { } fn transform_create_batch_request(&self) -> Result { - Err(Error::Unsupported("Anthropic message batch creation")) + Err(Error::Declined(Decline { + reason: "Anthropic message batch creation", + })) } fn transform_create_batch_response( @@ -135,7 +138,9 @@ impl AnthropicBatchesConfig for AnthropicBatchesTransformation { _response: AnthropicMessageBatch, _now: i64, ) -> Result { - Err(Error::Unsupported("Anthropic message batch creation")) + Err(Error::Declined(Decline { + reason: "Anthropic message batch creation", + })) } fn retrieve_batch_url( @@ -332,12 +337,16 @@ mod tests { fn preserves_python_placeholder_for_batch_creation() { assert!(matches!( ANTHROPIC_BATCHES_TRANSFORMATION.transform_create_batch_request(), - Err(Error::Unsupported("Anthropic message batch creation")) + Err(Error::Declined(Decline { + reason: "Anthropic message batch creation" + })) )); let response: AnthropicMessageBatch = serde_json::from_value(json!({})).unwrap(); assert!(matches!( ANTHROPIC_BATCHES_TRANSFORMATION.transform_create_batch_response(response, 0), - Err(Error::Unsupported("Anthropic message batch creation")) + Err(Error::Declined(Decline { + reason: "Anthropic message batch creation" + })) )); } } diff --git a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs index 91baa247ac8..7af37726520 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs @@ -1,4 +1,5 @@ use litellm_auth::SecretValue; +use litellm_core_utils::decline::Decline; use litellm_core_utils::{ core_helpers::{finish_reason_for, unix_now, usage_from_parts}, prompt_templates::factory::{Conversation, build_conversation}, @@ -27,7 +28,7 @@ use crate::{ streaming::{ChatStream, StreamShape}, transformation::{ BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, - Unsupported, ValidatedEnvironment, unsupported_message, unsupported_param, + ValidatedEnvironment, unsupported_message, unsupported_param, }, }, messages::streaming::event_stream as messages_event_stream, @@ -130,15 +131,14 @@ impl BaseConfig for AnthropicConfig { serde_json::from_value(response.body).map_err(|error| { Error::InvalidResponse(crate::ErrorDetail::invalid("messages response", error)) })?; - // The route declines tool and thinking requests, so a non-text block - // means the response carries something this path never asked for. - // Decline rather than silently dropping it; the host falls back. if body .content .iter() .any(|block| matches!(block, TextResponseBlock::Other)) { - return Err(Error::Unsupported("non-text response content block")); + return Err(Error::InvalidResponse( + "non-text response content block".into(), + )); } let text: String = body .content @@ -214,7 +214,7 @@ impl BaseConfig for AnthropicConfig { &self, messages: &[ChatMessage], optional_params: &Map, - ) -> Option { + ) -> Option { unsupported_param(self.supported_openai_param_mappings(), &[], optional_params) .or_else(|| messages.iter().find_map(unsupported_message)) // Anthropic rejects a request whose first turn is not a user turn. @@ -222,7 +222,7 @@ impl BaseConfig for AnthropicConfig { // core cannot observe, so decline instead of guessing. .or_else(|| { (!build_conversation(messages).opens_on_user_turn()) - .then_some(Unsupported("conversation does not open on a user turn")) + .then_some(Decline::new("conversation does not open on a user turn")) }) } } @@ -271,9 +271,10 @@ mod tests { anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, base_llm::{ auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, + chat::transformation::{BaseConfig, ProviderChatResponseData}, }, }; + use litellm_core_utils::decline::Decline; use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; use rstest::rstest; use serde_json::{Map, Value, json}; @@ -301,7 +302,7 @@ mod tests { .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) } - fn reason(msgs: Value, opts: Value) -> Option { + fn reason(msgs: Value, opts: Value) -> Option { ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) } @@ -409,7 +410,7 @@ mod tests { json!([{"role": "user", "content": "hi"}]), json!({"top_k": 40}) ), - Some(Unsupported("unrecognized request parameter")) + Some(Decline::new("unrecognized request parameter")) ); } @@ -420,7 +421,7 @@ mod tests { json!([{"role": "user", "content": "hi"}]), json!({"stream": true, "max_tokens": 16}) ), - Some(Unsupported("streaming")) + Some(Decline::new("streaming")) ); } @@ -445,7 +446,7 @@ mod tests { fn declines_any_param_outside_the_allowlist(#[case] param: Value) { assert_eq!( reason(json!([{"role": "user", "content": "hi"}]), param.clone()), - Some(Unsupported("unrecognized request parameter")), + Some(Decline::new("unrecognized request parameter")), "expected {param} to decline" ); } @@ -463,7 +464,7 @@ mod tests { ]), json!({}) ), - Some(Unsupported("unrecognized message field")) + Some(Decline::new("unrecognized message field")) ); assert_eq!( reason( @@ -473,7 +474,7 @@ mod tests { ]), json!({}) ), - Some(Unsupported("unrecognized message field")) + Some(Decline::new("unrecognized message field")) ); assert_eq!( reason( @@ -483,7 +484,7 @@ mod tests { ]}]), json!({}) ), - Some(Unsupported("non-text message content")) + Some(Decline::new("non-text message content")) ); assert_eq!( reason( @@ -493,7 +494,7 @@ mod tests { ]}]), json!({}) ), - Some(Unsupported("non-text message content")) + Some(Decline::new("non-text message content")) ); } @@ -504,7 +505,7 @@ mod tests { // than declining to Python before it. assert_eq!( reason(json!([{"role": "user", "content": []}]), json!({})), - Some(Unsupported("message without content")) + Some(Decline::new("message without content")) ); assert_eq!( reason( @@ -525,7 +526,7 @@ mod tests { ]), json!({}) ), - Some(Unsupported("conversation does not open on a user turn")) + Some(Decline::new("conversation does not open on a user turn")) ); } @@ -658,7 +659,7 @@ mod tests { } #[test] - fn declines_a_response_carrying_a_non_text_block() { + fn a_response_carrying_a_non_text_block_is_invalid() { let err = transform_response(json!({ "model": "claude-sonnet-4-5", "content": [{"type": "tool_use", "id": "t1", "name": "f", "input": {}}], @@ -666,7 +667,10 @@ mod tests { "usage": {"input_tokens": 1, "output_tokens": 1} })) .expect_err("non-text block"); - assert_eq!(err, Error::Unsupported("non-text response content block")); + assert_eq!( + err, + Error::InvalidResponse("non-text response content block".into()) + ); } #[rstest::rstest] diff --git a/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs index 30bff76651e..8ba4030540c 100644 --- a/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::decline::Decline; use litellm_llms_types::formats::chat_completions::{ ChatCompletionsResponse, ChatMessage, ChatMessageContent, }; @@ -29,16 +30,6 @@ const IGNORABLE_MESSAGE_FIELDS: &[&str] = &["name"]; pub use crate::base_llm::auth::{Headers, ValidatedEnvironment}; -/// Why a request cannot be served by the Rust path. -/// -/// The core declines rather than guessing: the host turns this into a -/// transparent fallback to the Python implementation, which covers the full -/// surface. Acceptance is an allowlist, so a parameter or message shape the -/// core has never seen declines by construction instead of being translated -/// wrong. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -pub struct Unsupported(pub &'static str); - pub trait BaseConfig: Sync { fn secret_names(&self) -> Vec<&'static str>; @@ -97,7 +88,7 @@ pub trait BaseConfig: Sync { &self, messages: &[ChatMessage], optional_params: &Map, - ) -> Option { + ) -> Option { unsupported_param( self.supported_openai_param_mappings(), self.config_params(), @@ -111,13 +102,13 @@ pub fn unsupported_param( supported: &'static [(&'static str, &'static str)], config: &'static [&'static str], optional_params: &Map, -) -> Option { +) -> Option { if optional_params .get(STREAM_PARAM) .and_then(Value::as_bool) .unwrap_or(false) { - return Some(Unsupported("streaming")); + return Some(Decline::new("streaming")); } optional_params .keys() @@ -128,29 +119,29 @@ pub fn unsupported_param( .any(|(_, provider_name)| *provider_name == key) && !config.contains(&key.as_str()) }) - .then_some(Unsupported("unrecognized request parameter")) + .then_some(Decline::new("unrecognized request parameter")) } /// Message shapes the core can translate faithfully: text content, either a /// plain string or a non-empty list of parts that are all /// `{"type": "text", "text": ...}`. Tool calls, tool results, and multimodal /// parts decline so Python's fuller translation handles them. -pub fn unsupported_message(message: &ChatMessage) -> Option { +pub fn unsupported_message(message: &ChatMessage) -> Option { if message .extra .keys() .any(|key| !IGNORABLE_MESSAGE_FIELDS.contains(&key.as_str())) { - return Some(Unsupported("unrecognized message field")); + return Some(Decline::new("unrecognized message field")); } if !matches!(message.role.as_str(), "system" | "user" | "assistant") { - return Some(Unsupported("unrecognized message role")); + return Some(Decline::new("unrecognized message role")); } match &message.content { - None => Some(Unsupported("message without content")), + None => Some(Decline::new("message without content")), Some(ChatMessageContent::Text(_)) => None, Some(ChatMessageContent::Parts(parts)) if parts.is_empty() => { - Some(Unsupported("message without content")) + Some(Decline::new("message without content")) } Some(ChatMessageContent::Parts(parts)) => parts .iter() @@ -159,6 +150,6 @@ pub fn unsupported_message(message: &ChatMessage) -> Option { || part.get("text").and_then(Value::as_str).is_none() || part.as_object().is_some_and(|object| object.len() != 2) }) - .then_some(Unsupported("non-text message content")), + .then_some(Decline::new("non-text message content")), } } diff --git a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs index 392dd56412c..6ea0f94c5f8 100644 --- a/litellm-rust/crates/llms/src/base_llm/ocr/error.rs +++ b/litellm-rust/crates/llms/src/base_llm/ocr/error.rs @@ -72,8 +72,6 @@ pub enum Error { PollOrigin, #[error("OCR polling timed out")] PollTimeout, - #[error("unsupported by the rust path: {0}")] - Unsupported(&'static str), #[error("invalid provider: {0}")] InvalidProvider(String), #[error("invalid model: {provider} has no model {model:?} - use one of: {}", supported.join(", "))] diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs index caa345a7024..6ddef24bf72 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -5,6 +5,7 @@ use litellm_auth_aws::{ constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, resolve_bedrock_region, }; +use litellm_core_utils::decline::Decline; use litellm_core_utils::{ core_helpers::{finish_reason_for, unix_now, usage_from_parts}, prompt_templates::factory::{Conversation, TurnRole, build_conversation}, @@ -26,7 +27,7 @@ use crate::{ streaming::StreamShape, transformation::{ BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, - Unsupported, ValidatedEnvironment, unsupported_message, unsupported_param, + ValidatedEnvironment, unsupported_message, unsupported_param, }, }, }, @@ -169,10 +170,10 @@ impl BaseConfig for AmazonConverseConfig { let response: ConverseResponse = serde_json::from_value(body).map_err(|error| { Error::InvalidResponse(format!("invalid Converse response: {error}").into()) })?; - // The route declines tool requests, so anything other than a text block - // is something this path never asked for. Decline; the host falls back. if response.message_content_is_non_text() { - return Err(Error::Unsupported("non-text response content block")); + return Err(Error::InvalidResponse( + "non-text response content block".into(), + )); } let text = response.content_text(); let computed = usage_from_parts( @@ -269,7 +270,7 @@ impl BaseConfig for AmazonConverseConfig { &self, messages: &[ChatMessage], optional_params: &Map, - ) -> Option { + ) -> Option { unsupported_param( self.supported_openai_param_mappings(), CONFIG_PARAMS, @@ -283,7 +284,7 @@ impl BaseConfig for AmazonConverseConfig { messages .iter() .any(has_blank_text) - .then_some(Unsupported("blank message text")) + .then_some(Decline::new("blank message text")) }) // Converse has no assistant prefill: Python inserts a continue turn // when a conversation opens or closes on an assistant message, and @@ -296,7 +297,7 @@ impl BaseConfig for AmazonConverseConfig { .turns .last() .is_some_and(|turn| turn.role == TurnRole::Assistant); - (!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Unsupported( + (!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Decline::new( "conversation does not run user turn to user turn", )) }) @@ -358,11 +359,12 @@ mod tests { Error, base_llm::{ auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, + chat::transformation::{BaseConfig, ProviderChatResponseData}, }, bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, }; use litellm_auth::CredentialPlacement; + use litellm_core_utils::decline::Decline; use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; use rstest::rstest; use serde_json::{Map, Value, json}; @@ -396,7 +398,7 @@ mod tests { ) } - fn reason(msgs: Value, opts: Value) -> Option { + fn reason(msgs: Value, opts: Value) -> Option { BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) } @@ -472,7 +474,7 @@ mod tests { json!([{"role": "user", "content": "hi"}]), json!({"stream": true}) ), - Some(Unsupported("streaming")) + Some(Decline::new("streaming")) ); } @@ -483,7 +485,7 @@ mod tests { json!([{"role": "user", "content": "hi"}]), json!({"topK": 40}) ), - Some(Unsupported("unrecognized request parameter")) + Some(Decline::new("unrecognized request parameter")) ); } @@ -497,7 +499,7 @@ mod tests { fn declines_tools_and_other_params_outside_the_allowlist(#[case] param: Value) { assert_eq!( reason(json!([{"role": "user", "content": "hi"}]), param.clone()), - Some(Unsupported("unrecognized request parameter")), + Some(Decline::new("unrecognized request parameter")), "expected {param} to decline" ); } @@ -514,7 +516,7 @@ mod tests { json!([{"role": "user", "content": content}, {"role": "user", "content": "hi"}]), json!({}) ), - Some(Unsupported("blank message text")), + Some(Decline::new("blank message text")), "expected blank content {content} to decline" ); } @@ -526,7 +528,7 @@ mod tests { // host before the call rather than an error after it. assert_eq!( reason(json!([{"role": "user", "content": []}]), json!({})), - Some(Unsupported("message without content")) + Some(Decline::new("message without content")) ); assert_eq!( reason( @@ -547,7 +549,7 @@ mod tests { ]), json!({}) ), - Some(Unsupported( + Some(Decline::new( "conversation does not run user turn to user turn" )) ); @@ -559,7 +561,7 @@ mod tests { ]), json!({}) ), - Some(Unsupported( + Some(Decline::new( "conversation does not run user turn to user turn" )) ); @@ -819,7 +821,7 @@ mod tests { ]}]), json!({}) ), - Some(Unsupported("non-text message content")) + Some(Decline::new("non-text message content")) ); } @@ -846,7 +848,7 @@ mod tests { } #[test] - fn declines_a_response_carrying_a_tool_use_block() { + fn a_response_carrying_a_tool_use_block_is_invalid() { let err = transform_response(json!({ "output": {"message": {"content": [ {"toolUse": {"toolUseId": "t1", "name": "f", "input": {}}} @@ -855,7 +857,10 @@ mod tests { "usage": {"inputTokens": 1, "outputTokens": 1} })) .expect_err("tool use block"); - assert_eq!(err, Error::Unsupported("non-text response content block")); + assert_eq!( + err, + Error::InvalidResponse("non-text response content block".into()) + ); } #[test] diff --git a/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs b/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs index 878c424ae2d..342b5e935b2 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs @@ -1,6 +1,7 @@ use base64::Engine; use bytes::Buf; use futures_util::{Stream, StreamExt}; +use litellm_core_utils::decline::Decline; use litellm_framer::{ aws_event_stream::{AwsEventStreamCodec, Message}, frames, @@ -85,10 +86,12 @@ pub fn invoke_chat_stream(invoke_provider: &str, shape: StreamShape) -> Result Err(Error::Unsupported( - "Bedrock invoke streaming for this model family", + InvokeProvider::DeepseekR1 | InvokeProvider::Moonshot => Err(Error::Declined( + Decline::new("Bedrock invoke streaming for this model family"), )), - InvokeProvider::Unsupported => Err(Error::Unsupported("Bedrock invoke streaming")), + InvokeProvider::Unsupported => { + Err(Error::Declined(Decline::new("Bedrock invoke streaming"))) + } } } diff --git a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs index 1f6b6edf28d..3ec60f12021 100644 --- a/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::decline::Decline; use std::convert::Infallible; use crate::{ @@ -130,9 +131,9 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { _request: MessagesRequest, _context: &MessagesTransformContext, ) -> Result { - Err(Error::Unsupported( + Err(Error::Declined(Decline::new( "Bedrock invoke messages request shaping", - )) + ))) } fn secret_names(&self) -> &'static [&'static str] { diff --git a/litellm-rust/crates/llms/src/error.rs b/litellm-rust/crates/llms/src/error.rs index 10a4f696406..01b2e916a8a 100644 --- a/litellm-rust/crates/llms/src/error.rs +++ b/litellm-rust/crates/llms/src/error.rs @@ -1,3 +1,5 @@ +use litellm_core_utils::decline::Decline; + #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] pub enum Error { #[error("expected {expected}, got {actual}")] @@ -11,8 +13,8 @@ pub enum Error { InvalidRequest(#[source] ErrorDetail), #[error("invalid response: {0}")] InvalidResponse(#[source] ErrorDetail), - #[error("unsupported: {0}")] - Unsupported(&'static str), + #[error(transparent)] + Declined(Decline), #[error(transparent)] Auth(#[from] litellm_auth::Error), } diff --git a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs index 0a89adf90f7..cb0be6cee72 100644 --- a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs @@ -243,14 +243,14 @@ fn normalize_choice(position: usize, choice: &Value) -> Result Option { + fn reason(msgs: Value, opts: Value) -> Option { OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) } @@ -528,9 +529,7 @@ mod tests { } #[rstest] - fn a_tool_call_response_declines_instead_of_dropping_the_calls() { - // The `json_mode` rewrite needs a request flag the route does not carry, so - // a tool-call answer falls back to Python rather than losing the calls. + fn a_tool_call_response_fails_instead_of_dropping_the_calls() { assert_eq!( transform_response(json!({ "model": "m", @@ -547,12 +546,12 @@ mod tests { "finish_reason": "tool_calls", }], })), - Err(Error::Unsupported("tool call response")) + Err(Error::InvalidResponse("tool call response".into())) ); } #[rstest] - fn a_refusal_declines_instead_of_returning_an_empty_reply() { + fn a_refusal_fails_instead_of_returning_an_empty_reply() { assert_eq!( transform_response(json!({ "model": "m", @@ -561,12 +560,12 @@ mod tests { "finish_reason": "stop", }], })), - Err(Error::Unsupported("refusal response")) + Err(Error::InvalidResponse("refusal response".into())) ); } #[rstest] - fn a_non_text_response_content_declines() { + fn a_non_text_response_content_is_invalid() { assert_eq!( transform_response(json!({ "model": "m", @@ -575,7 +574,7 @@ mod tests { "finish_reason": "stop", }], })), - Err(Error::Unsupported("non-text response content")) + Err(Error::InvalidResponse("non-text response content".into())) ); } @@ -585,7 +584,7 @@ mod tests { fn declines(#[case] opts: Value, #[case] expected: &'static str) { assert_eq!( reason(json!([{"role": "user", "content": "hi"}]), opts), - Some(Unsupported(expected)) + Some(Decline::new(expected)) ); } @@ -615,7 +614,7 @@ mod tests { json!([{"role": "user", "content": "hi"}]), json!({"tools": [{"type": "function", "function": {"name": "f"}}]}), ), - Some(Unsupported("unrecognized request parameter")) + Some(Decline::new("unrecognized request parameter")) ); } } diff --git a/litellm-rust/crates/python-bridge/src/errors.rs b/litellm-rust/crates/python-bridge/src/errors.rs index 4dab0297e73..9c0b1544570 100644 --- a/litellm-rust/crates/python-bridge/src/errors.rs +++ b/litellm-rust/crates/python-bridge/src/errors.rs @@ -20,6 +20,9 @@ pyo3::create_exception!( ); pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr { + if let Some(declined) = decline_to_pyerr(&error) { + return declined; + } match error { Error::Transport(TransportError::Http { status, body }) => { RustUpstreamError::new_err((status, body)) @@ -28,6 +31,14 @@ pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr { } } +/// A core decline skips the route's public exception mapping, so Python dispatch catches it +/// and runs the legacy path. +pub(crate) fn decline_to_pyerr(error: &Error) -> Option { + error + .declined() + .map(|decline| RustBridgeDeclined::new_err(decline.reason)) +} + /// A request the caller got wrong is a `ValueError`; anything else is a `RuntimeError`. pub(crate) fn by_fault(is_request: bool, message: String) -> PyErr { if is_request { @@ -39,10 +50,11 @@ pub(crate) fn by_fault(is_request: bool, message: String) -> PyErr { #[cfg(test)] mod tests { + use litellm_core_utils::decline::Decline; + use super::*; #[rstest::rstest] - #[case::unsupported(Error::Unsupported("test capability"), true)] #[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)] #[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)] #[case::connection(TransportError::Connect("unreachable".into()).into(), false)] @@ -58,6 +70,26 @@ mod tests { }); } + #[test] + fn a_core_decline_hands_the_call_back_to_python() { + Python::initialize(); + Python::attach(|py| { + let declined = route_error_to_pyerr(Error::from(litellm_llms::Error::Declined( + Decline::new("unrecognized request parameter"), + ))); + assert!(declined.is_instance_of::(py)); + assert_eq!( + declined + .value(py) + .getattr("args") + .unwrap() + .extract::<(String,)>() + .unwrap(), + ("unrecognized request parameter".into(),) + ); + }); + } + #[rstest::rstest] fn transport_status_survives_python_mapping() { Python::initialize(); diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs index 5ad9f8df8b8..84cfd44f52e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::decline::Decline; use std::convert::Infallible; use crate::routes::inference::InferenceHost; @@ -44,9 +45,9 @@ impl PythonBinding for ChatCompletionsPythonHost { .get("stream") .is_some_and(|value| value == &serde_json::Value::Bool(true)) { - return Err(InvokeError::Native(Error::Unsupported( + return Err(InvokeError::Native(Error::Declined(Decline::new( "native Python chat_completions streaming", - ))); + )))); } Ok(call) } diff --git a/litellm-rust/crates/python-bridge/src/routes/inference.rs b/litellm-rust/crates/python-bridge/src/routes/inference.rs index c9eb60e47ed..3409a824b11 100644 --- a/litellm-rust/crates/python-bridge/src/routes/inference.rs +++ b/litellm-rust/crates/python-bridge/src/routes/inference.rs @@ -20,7 +20,7 @@ use serde_json::{Map, Value}; use super::{NativeCall, RequestView}; use crate::{ - errors::{RustUpstreamError, route_error_to_pyerr}, + errors::{RustUpstreamError, decline_to_pyerr, route_error_to_pyerr}, marshal::{RouteOptions, optional_timeout, public_response, python_timeout_seconds}, }; @@ -120,6 +120,9 @@ impl InferenceHost { } pub fn error(&self, py: Python<'_>, error: Error) -> PyResult { + if let Some(declined) = decline_to_pyerr(&error) { + return Ok(declined); + } if let Error::Secret(source) = &error && let Some(original) = crate::secrets::python_error(py, source.source_error()) { diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 5e445fcaa7d..27323da4c4e 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -20,7 +20,7 @@ use pyo3::{ use serde_json::{Map, Value}; use crate::{ - errors::{RustUpstreamError, route_error_to_pyerr}, + errors::{RustUpstreamError, decline_to_pyerr, route_error_to_pyerr}, marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds}, python_settings::missing_module, routes::RequestView, @@ -276,6 +276,9 @@ impl PythonBinding for MessagesPythonHost { } fn map_error(&self, py: Python<'_>, error: Error) -> PyResult { + if let Some(declined) = decline_to_pyerr(&error) { + return Ok(declined); + } if let Error::Secret(source) = &error && let Some(original) = crate::secrets::python_error(py, source.source_error()) { diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index 9c430ba7868..9390c596770 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -1,3 +1,4 @@ +use litellm_core_utils::decline::Decline; use std::convert::Infallible; use crate::routes::inference::InferenceHost; @@ -118,9 +119,9 @@ impl PythonBinding for ResponsesPythonHost { .get("stream") .is_some_and(|value| value == &serde_json::Value::Bool(true)) { - return Err(InvokeError::Native(Error::Unsupported( + return Err(InvokeError::Native(Error::Declined(Decline::new( "native Python responses streaming", - ))); + )))); } Ok(call) }