scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-29 06:33:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 200

feat(rust): one typed decline from core up to each host

Browse source 
Core used Unsupported(&str) in two enums plus a chat-only Unsupported struct, and
the SDK bridge turned a core decline into a terminal ValueError instead of falling
back. Some response transforms also raised Unsupported after the provider answered

A Decline in core-utils now means "nothing was sent, another host may serve this
as is". llms::Error and inference::Error carry it as Declined, the From impl keeps
it, and inference::Error::declined() is an exhaustive match. after_send() moves
from the chat handler into inference so every route collapses errors raised while
normalizing a response, which keeps a decline from ever leaving after the send.
Response transforms that used Unsupported now report InvalidResponse

The bridge maps a decline to RustBridgeDeclined before the route's public
exception mapping, so runtime.py falls back to legacy Python. OCR's Unsupported
was never raised and is gone
This commit is contained in:
Yujong Lee 2026-10-20 19:34:35 -07:00 • committed by GitHub
parent dc044432cf
commit a3ae1be8f2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
28 changed files with 269 additions and 164 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
litellm-rust/crates/core-utils/src/decline.rs Normal file
View file

@ -0,0 +1,16 @@
/// Rust core cannot serve this request, and nothing was sent to the provider yet.
///
/// Every error on the call path carries it unchanged, so the host can hand the same request
/// to another implementation: the SDK runs legacy Python, the gateway forwards to the proxy.
/// Raise it only before the provider call. Anything after the send is terminal.
#[derive(Clone, Copy, Debug, PartialEq, Eq, thiserror::Error)]
#[error("declined by the rust path: {reason}")]
pub struct Decline {
pub reason: &'static str,
}
impl Decline {
pub const fn new(reason: &'static str) -> Self {
Self { reason }
}
}

1
litellm-rust/crates/core-utils/src/lib.rs
View file

@ -1,5 +1,6 @@
pub mod call_arguments;
pub mod core_helpers;
pub mod decline;
pub mod dot_notation_indexing;
pub mod exception_mapping_utils;
pub mod get_llm_provider_logic;

5
litellm-rust/crates/gateway-inference/src/error.rs
View file

@ -49,9 +49,8 @@ impl Error {
pub fn status(&self) -> StatusCode {
match self {
Self::Auth(error) => error.status(),
Self::Unsupported(_)
| Self::Route(InferenceError::Unsupported(_))
| Self::Ocr(OcrError::Unsupported(_)) => StatusCode::NOT_IMPLEMENTED,
Self::Unsupported(_) => StatusCode::NOT_IMPLEMENTED,
Self::Route(error) if error.declined().is_some() => StatusCode::NOT_IMPLEMENTED,
Self::BodyTooLarge => StatusCode::PAYLOAD_TOO_LARGE,
Self::Ocr(
OcrError::Auth(litellm_auth::Error::MissingApiKey { .. })

2
litellm-rust/crates/inference-chat/src/common_utils.rs
View file

@ -61,7 +61,7 @@ pub(super) fn resolve_request(
));
}
if let Some(reason) = config.unsupported_reason(&messages, &call.optional_params) {
return Err(Error::Unsupported(reason.0));
return Err(Error::Declined(reason));
}
Ok(ResolvedChatCompletionsRequest {
messages,

43
litellm-rust/crates/inference-chat/src/handler.rs
View file

@ -2,6 +2,7 @@ use litellm_host::{lifecycle::ExecutionEvent, observation::ObservationSender};
use std::time::Duration;
use litellm_auth::AuthServices;
use litellm_core_utils::decline::Decline;
use litellm_host::interceptors::{Interceptors, RawResponse, RequestContext, WireRequest};
use litellm_http::{
Client,
@ -145,29 +146,12 @@ pub(super) async fn execute(
config
.transform_response(provider.model, ProviderChatResponseData { body })
.map_err(Error::from)
.map_err(as_response_error)
.map_err(Error::after_send)
},
)
.await
}
/// Re-tag an error raised while normalizing a response the provider already
/// returned.
///
/// A config reports the same variants on either side of the call: a missing
/// field or an unsupported block can mean "this request cannot be translated"
/// before the request is sent and "this response cannot be normalized" here. Only the
/// second kind has already been billed, and a host that keeps a reference
/// implementation must not retry those, so collapse them to one variant that
/// can only mean the provider was already called.
pub(super) fn as_response_error(err: Error) -> Error {
match err {
already @ (Error::InvalidResponse(_)
| Error::Transport(litellm_http::transport::Error::Http { .. })) => already,
other => Error::InvalidResponse(other.to_string().into()),
}
}
pub(super) fn outbound_request(
authenticated: Authenticated,
url: String,
@ -184,7 +168,7 @@ pub(super) fn outbound_request(
// Python drops the caller's copy and prefers a forwarded Authorization
// over the signature, so leave the request to it.
litellm_http::Error::ComputedHeader(_) => {
Error::Unsupported("request forwards a header AWS SigV4 computes")
Error::Declined(Decline::new("request forwards a header AWS SigV4 computes"))
}
other => Error::Http(other),
})
@ -348,25 +332,4 @@ mod tests {
));
assert!(interceptors.raw.into_inner().unwrap().is_empty());
}
#[rstest::rstest]
fn response_errors_collapse_to_one_variant_that_can_only_mean_already_sent() {
for original in [
Error::MissingField("usage"),
Error::Unsupported("non-text response content block"),
Error::InvalidRequest("whatever".to_string().into()),
Error::Auth(litellm_auth::Error::InvalidHeader),
] {
let label = format!("{original:?}");
assert!(
matches!(as_response_error(original), Error::InvalidResponse(_)),
"{label} must not stay retryable once the provider has answered"
);
}
let upstream = Error::Transport(litellm_http::transport::Error::Http {
status: 500,
body: "boom".to_string(),
});
assert_eq!(as_response_error(upstream.clone()), upstream);
}
}

2
litellm-rust/crates/inference-chat/tests/request.rs
View file

@ -355,7 +355,7 @@ async fn rejects_a_forwarded_header_the_signer_computes(#[case] forwarded: &str)
let error = complete(call)
.await
.expect_err("conflicting signing headers fail");
assert!(matches!(error, Error::Unsupported(_)), "{error:?}");
assert!(error.declined().is_some(), "{error:?}");
assert!(received(&upstream).await.is_empty());
}

1
litellm-rust/crates/inference-messages/src/handler.rs
View file

@ -325,6 +325,7 @@ fn decode_response(
config
.transform_anthropic_messages_response(model, response)
.map_err(Error::from)
.map_err(Error::after_send)
}
fn streaming_response(

4
litellm-rust/crates/inference-messages/tests/messages/host.rs
View file

@ -81,7 +81,7 @@ impl litellm_host::interceptors::Interceptors<<Messages as litellm_host::protoco
async fn result_ready(&self, facts: ExecutionFacts) -> Result<(), Error> {
self.facts.lock().unwrap().push(facts);
if self.reject_result {
return Err(Error::Unsupported("result rejected"));
return Err(Error::InvalidRequest("result rejected".into()));
}
Ok(())
}
@ -181,7 +181,7 @@ async fn rejected_results_are_not_delivered_or_cached(
assert_eq!(
result,
if reject {
Err(Error::Unsupported("result rejected"))
Err(Error::InvalidRequest("result rejected".into()))
} else {
Ok(())
}

1
litellm-rust/crates/inference-responses/src/handler.rs
View file

@ -129,6 +129,7 @@ pub(super) async fn execute(
.transform_response_api_response(value)
.map(ResponsesOutput::Complete)
.map_err(Error::from)
.map_err(Error::after_send)
},
)
.await

10
litellm-rust/crates/inference-responses/src/lib.rs
View file

@ -11,6 +11,7 @@ mod handler;
use std::sync::Arc;
use litellm_auth::AuthServices;
use litellm_core_utils::decline::Decline;
use litellm_core_utils::get_llm_provider_logic::{
CustomLlmProvider, LlmProviders, get_custom_llm_provider,
};
@ -24,6 +25,8 @@ use litellm_llms::{
use litellm_secrets::source::SecretSource;
use types::{ResponsesCall, ResponsesOutput, ResponsesProvider};
const NOT_NATIVE: Decline = Decline::new("native HTTP responses provider");
fn responses_provider(provider: LlmProviders) -> Option<&'static dyn BaseResponsesApiConfig> {
match provider {
LlmProviders::Openai => Some(&OpenAiResponsesApiConfig),
@ -111,9 +114,8 @@ impl ResponsesRoute {
}
let provider = provider_name
.parse::<LlmProviders>()
.map_err(|_| Error::Unsupported("native HTTP responses provider"))?;
let config = responses_provider(provider)
.ok_or(Error::Unsupported("native HTTP responses provider"))?;
.map_err(|_| Error::Declined(NOT_NATIVE))?;
let config = responses_provider(provider).ok_or(Error::Declined(NOT_NATIVE))?;
let wire_model = match provider {
LlmProviders::Openai if resolved_model.contains('/') => {
return Err(Error::InvalidProvider(call.model));
@ -123,7 +125,7 @@ impl ResponsesRoute {
format!("openrouter/{resolved_model}")
}
LlmProviders::Openrouter => resolved_model.into(),
_ => return Err(Error::Unsupported("native HTTP responses provider")),
_ => return Err(Error::Declined(NOT_NATIVE)),
};
let provider = ResponsesProvider {
identity: ProviderIdentity {

5
litellm-rust/crates/inference-responses/tests/responses.rs
View file

@ -103,10 +103,7 @@ async fn unsupported_providers_fail_before_transport(
)
.await;
assert!(matches!(
result,
Err(Error::Unsupported("native HTTP responses provider"))
));
assert!(matches!(result, Err(ref error) if error.declined().is_some()));
assert!(received(&upstream).await.is_empty());
}

7
litellm-rust/crates/inference-transcription/src/handler.rs
View file

@ -76,7 +76,8 @@ pub(super) async fn execute(
error,
))
})?;
Ok(config
.transform_audio_transcription_response(provider.model, response_json)?
.into_json())
config
.transform_audio_transcription_response(provider.model, response_json)
.map(|response| response.into_json())
.map_err(|error| Error::from(error).after_send())
}

81
litellm-rust/crates/inference/src/error.rs
View file

@ -7,6 +7,7 @@
use std::sync::Arc;
use litellm_core_utils::decline::Decline;
use litellm_http::transport::Error as TransportError;
use litellm_llms::Error as LlmError;
@ -25,8 +26,8 @@ pub enum Error {
InvalidRequest(#[source] litellm_llms::ErrorDetail),
#[error("invalid response: {0}")]
InvalidResponse(#[source] litellm_llms::ErrorDetail),
#[error("unsupported by the rust path: {0}")]
Unsupported(&'static str),
#[error(transparent)]
Declined(Decline),
#[error(transparent)]
Auth(#[from] litellm_auth::Error),
#[error(transparent)]
@ -64,16 +65,50 @@ impl Error {
| Self::MissingField(_)
| Self::InvalidProvider(_)
| Self::InvalidRequest(_)
| Self::Unsupported(_)
| Self::Headers(_) => true,
Self::Auth(error) => !matches!(error, litellm_auth::Error::MissingApiKey { .. }),
Self::InvalidResponse(_)
Self::Declined(_)
| Self::InvalidResponse(_)
| Self::Transport(_)
| Self::Http(_)
| Self::Secret(_)
| Self::PostCallHook(_) => false,
}
}
/// The decline this error carries, if the request never reached the provider.
pub fn declined(&self) -> Option<&Decline> {
match self {
Self::Declined(decline) => Some(decline),
Self::InvalidType { .. }
| Self::MissingField(_)
| Self::InvalidProvider(_)
| Self::InvalidRequest(_)
| Self::InvalidResponse(_)
| Self::Auth(_)
| Self::Transport(_)
| Self::Headers(_)
| Self::Http(_)
| Self::Secret(_)
| Self::PostCallHook(_) => None,
}
}
/// Re-tag an error raised while normalizing a response the provider already returned.
///
/// A config reports the same variants on either side of the call: a missing field can
/// mean "this request cannot be translated" before the send and "this response cannot be
/// normalized" after it. Only the second kind was billed, and a host that keeps another
/// implementation must not retry it, so collapse it to the one variant that can only mean
/// the provider was already called.
pub fn after_send(self) -> Self {
match self {
already @ (Self::InvalidResponse(_) | Self::Transport(TransportError::Http { .. })) => {
already
}
other => Self::InvalidResponse(other.to_string().into()),
}
}
}
impl From<LlmError> for Error {
@ -83,7 +118,7 @@ impl From<LlmError> for Error {
LlmError::MissingField(field) => Self::MissingField(field),
LlmError::InvalidRequest(message) => Self::InvalidRequest(message),
LlmError::InvalidResponse(message) => Self::InvalidResponse(message),
LlmError::Unsupported(reason) => Self::Unsupported(reason),
LlmError::Declined(decline) => Self::Declined(decline),
LlmError::Auth(error) => Self::Auth(error),
}
}
@ -116,9 +151,45 @@ impl Eq for SecretError {}
#[cfg(test)]
mod tests {
use super::Error;
use litellm_core_utils::decline::Decline;
use litellm_llms::{Error as LlmError, ErrorDetail};
use rstest::rstest;
#[test]
fn a_transformation_decline_survives_into_the_route_error() {
let decline = Decline::new("unrecognized request parameter");
assert_eq!(
Error::from(LlmError::Declined(decline)).declined(),
Some(&decline)
);
}
#[rstest]
#[case::declined(Error::Declined(Decline::new("non-text response content block")))]
#[case::missing_field(Error::MissingField("usage"))]
#[case::invalid_request(Error::InvalidRequest("whatever".into()))]
#[case::auth(Error::Auth(litellm_auth::Error::InvalidHeader))]
fn errors_after_the_send_are_never_a_decline(#[case] error: Error) {
let collapsed = error.after_send();
assert!(matches!(collapsed, Error::InvalidResponse(_)));
assert_eq!(collapsed.declined(), None);
}
#[test]
fn an_upstream_status_stays_as_is_after_the_send() {
let upstream = Error::Transport(litellm_http::transport::Error::Http {
status: 500,
body: "boom".into(),
});
assert_eq!(upstream.clone().after_send(), upstream);
}
#[test]
fn a_post_call_hook_failure_is_never_a_decline() {
let hook = Error::post_call(Error::Declined(Decline::new("hook")));
assert_eq!(hook.declined(), None);
}
#[test]
fn a_missing_api_key_is_the_environment_not_the_request() {
assert!(

15
litellm-rust/crates/inference/tests/caching.rs
View file

@ -204,7 +204,8 @@ async fn streamed(
.chunks(3)
.map(|bytes| Ok(Bytes::copy_from_slice(bytes)))
.collect::<Vec<_>>();
let ending = fail.then_some(Err(Error::Unsupported("test transport failure")));
let ending =
fail.then_some(Err(Error::InvalidRequest("test transport failure".into())));
Ok(CallOutput::Stream {
head: (),
chunks: stream::iter(chunks.into_iter().chain(ending)).boxed(),
@ -315,7 +316,7 @@ async fn a_provider_failure_never_populates_the_cache(cache: Arc<dyn ResponseCac
None,
|| async {
calls.fetch_add(1, Ordering::SeqCst);
Err(Error::Unsupported("test provider failure"))
Err(Error::InvalidRequest("test provider failure".into()))
},
)
.await;
@ -482,7 +483,7 @@ impl Interceptors<Error> for CacheHitAccounting {
);
self.key.set(key).unwrap();
if self.reject {
return Err(Error::Unsupported("cache accounting rejected"));
return Err(Error::InvalidRequest("cache accounting rejected".into()));
}
Ok(())
}
@ -569,10 +570,10 @@ async fn cache_hits_notify_accounting_once_and_propagate_its_failure(
.await
};
if reject {
assert!(matches!(
result,
Err(Error::Unsupported("cache accounting rejected"))
));
assert_eq!(
result.unwrap_err(),
Error::InvalidRequest("cache accounting rejected".into())
);
} else {
assert_eq!(result.unwrap(), expected);
}

17
litellm-rust/crates/llms/src/anthropic/batches/transformation.rs
View file

@ -1,3 +1,4 @@
use litellm_core_utils::decline::Decline;
use litellm_llms_types::formats::batches::{BatchRequestCounts, BatchResponse, BatchStatus};
use litellm_llms_types::formats::messages::MessagesResponse;
use litellm_llms_types::providers::anthropic::{BATCHES_PATH, MESSAGES_PATH};
@ -127,7 +128,9 @@ impl AnthropicBatchesConfig for AnthropicBatchesTransformation {
}
fn transform_create_batch_request(&self) -> Result<Value, Error> {
Err(Error::Unsupported("Anthropic message batch creation"))
Err(Error::Declined(Decline {
reason: "Anthropic message batch creation",
}))
}
fn transform_create_batch_response(
@ -135,7 +138,9 @@ impl AnthropicBatchesConfig for AnthropicBatchesTransformation {
_response: AnthropicMessageBatch,
_now: i64,
) -> Result<BatchResponse, Error> {
Err(Error::Unsupported("Anthropic message batch creation"))
Err(Error::Declined(Decline {
reason: "Anthropic message batch creation",
}))
}
fn retrieve_batch_url(
@ -332,12 +337,16 @@ mod tests {
fn preserves_python_placeholder_for_batch_creation() {
assert!(matches!(
ANTHROPIC_BATCHES_TRANSFORMATION.transform_create_batch_request(),
Err(Error::Unsupported("Anthropic message batch creation"))
Err(Error::Declined(Decline {
reason: "Anthropic message batch creation"
}))
));
let response: AnthropicMessageBatch = serde_json::from_value(json!({})).unwrap();
assert!(matches!(
ANTHROPIC_BATCHES_TRANSFORMATION.transform_create_batch_response(response, 0),
Err(Error::Unsupported("Anthropic message batch creation"))
Err(Error::Declined(Decline {
reason: "Anthropic message batch creation"
}))
));
}
}

44
litellm-rust/crates/llms/src/anthropic/chat/transformation.rs
View file

@ -1,4 +1,5 @@
use litellm_auth::SecretValue;
use litellm_core_utils::decline::Decline;
use litellm_core_utils::{
core_helpers::{finish_reason_for, unix_now, usage_from_parts},
prompt_templates::factory::{Conversation, build_conversation},
@ -27,7 +28,7 @@ use crate::{
streaming::{ChatStream, StreamShape},
transformation::{
BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData,
Unsupported, ValidatedEnvironment, unsupported_message, unsupported_param,
ValidatedEnvironment, unsupported_message, unsupported_param,
},
},
messages::streaming::event_stream as messages_event_stream,
@ -130,15 +131,14 @@ impl BaseConfig for AnthropicConfig {
serde_json::from_value(response.body).map_err(|error| {
Error::InvalidResponse(crate::ErrorDetail::invalid("messages response", error))
})?;
// The route declines tool and thinking requests, so a non-text block
// means the response carries something this path never asked for.
// Decline rather than silently dropping it; the host falls back.
if body
.content
.iter()
.any(|block| matches!(block, TextResponseBlock::Other))
{
return Err(Error::Unsupported("non-text response content block"));
return Err(Error::InvalidResponse(
"non-text response content block".into(),
));
}
let text: String = body
.content
@ -214,7 +214,7 @@ impl BaseConfig for AnthropicConfig {
&self,
messages: &[ChatMessage],
optional_params: &Map<String, Value>,
) -> Option<Unsupported> {
) -> Option<Decline> {
unsupported_param(self.supported_openai_param_mappings(), &[], optional_params)
.or_else(|| messages.iter().find_map(unsupported_message))
// Anthropic rejects a request whose first turn is not a user turn.
@ -222,7 +222,7 @@ impl BaseConfig for AnthropicConfig {
// core cannot observe, so decline instead of guessing.
.or_else(|| {
(!build_conversation(messages).opens_on_user_turn())
.then_some(Unsupported("conversation does not open on a user turn"))
.then_some(Decline::new("conversation does not open on a user turn"))
})
}
}
@ -271,9 +271,10 @@ mod tests {
anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG,
base_llm::{
auth::AuthScheme,
chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported},
chat::transformation::{BaseConfig, ProviderChatResponseData},
},
};
use litellm_core_utils::decline::Decline;
use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage};
use rstest::rstest;
use serde_json::{Map, Value, json};
@ -301,7 +302,7 @@ mod tests {
.transform_response("claude-sonnet-4-5", ProviderChatResponseData { body })
}
fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
fn reason(msgs: Value, opts: Value) -> Option<Decline> {
ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), &params(opts))
}
@ -409,7 +410,7 @@ mod tests {
json!([{"role": "user", "content": "hi"}]),
json!({"top_k": 40})
),
Some(Unsupported("unrecognized request parameter"))
Some(Decline::new("unrecognized request parameter"))
);
}
@ -420,7 +421,7 @@ mod tests {
json!([{"role": "user", "content": "hi"}]),
json!({"stream": true, "max_tokens": 16})
),
Some(Unsupported("streaming"))
Some(Decline::new("streaming"))
);
}
@ -445,7 +446,7 @@ mod tests {
fn declines_any_param_outside_the_allowlist(#[case] param: Value) {
assert_eq!(
reason(json!([{"role": "user", "content": "hi"}]), param.clone()),
Some(Unsupported("unrecognized request parameter")),
Some(Decline::new("unrecognized request parameter")),
"expected {param} to decline"
);
}
@ -463,7 +464,7 @@ mod tests {
]),
json!({})
),
Some(Unsupported("unrecognized message field"))
Some(Decline::new("unrecognized message field"))
);
assert_eq!(
reason(
@ -473,7 +474,7 @@ mod tests {
]),
json!({})
),
Some(Unsupported("unrecognized message field"))
Some(Decline::new("unrecognized message field"))
);
assert_eq!(
reason(
@ -483,7 +484,7 @@ mod tests {
]}]),
json!({})
),
Some(Unsupported("non-text message content"))
Some(Decline::new("non-text message content"))
);
assert_eq!(
reason(
@ -493,7 +494,7 @@ mod tests {
]}]),
json!({})
),
Some(Unsupported("non-text message content"))
Some(Decline::new("non-text message content"))
);
}
@ -504,7 +505,7 @@ mod tests {
// than declining to Python before it.
assert_eq!(
reason(json!([{"role": "user", "content": []}]), json!({})),
Some(Unsupported("message without content"))
Some(Decline::new("message without content"))
);
assert_eq!(
reason(
@ -525,7 +526,7 @@ mod tests {
]),
json!({})
),
Some(Unsupported("conversation does not open on a user turn"))
Some(Decline::new("conversation does not open on a user turn"))
);
}
@ -658,7 +659,7 @@ mod tests {
}
#[test]
fn declines_a_response_carrying_a_non_text_block() {
fn a_response_carrying_a_non_text_block_is_invalid() {
let err = transform_response(json!({
"model": "claude-sonnet-4-5",
"content": [{"type": "tool_use", "id": "t1", "name": "f", "input": {}}],
@ -666,7 +667,10 @@ mod tests {
"usage": {"input_tokens": 1, "output_tokens": 1}
}))
.expect_err("non-text block");
assert_eq!(err, Error::Unsupported("non-text response content block"));
assert_eq!(
err,
Error::InvalidResponse("non-text response content block".into())
);
}
#[rstest::rstest]

31
litellm-rust/crates/llms/src/base_llm/chat/transformation.rs
View file

@ -1,3 +1,4 @@
use litellm_core_utils::decline::Decline;
use litellm_llms_types::formats::chat_completions::{
ChatCompletionsResponse, ChatMessage, ChatMessageContent,
};
@ -29,16 +30,6 @@ const IGNORABLE_MESSAGE_FIELDS: &[&str] = &["name"];
pub use crate::base_llm::auth::{Headers, ValidatedEnvironment};
/// Why a request cannot be served by the Rust path.
///
/// The core declines rather than guessing: the host turns this into a
/// transparent fallback to the Python implementation, which covers the full
/// surface. Acceptance is an allowlist, so a parameter or message shape the
/// core has never seen declines by construction instead of being translated
/// wrong.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Unsupported(pub &'static str);
pub trait BaseConfig: Sync {
fn secret_names(&self) -> Vec<&'static str>;
@ -97,7 +88,7 @@ pub trait BaseConfig: Sync {
&self,
messages: &[ChatMessage],
optional_params: &Map<String, Value>,
) -> Option<Unsupported> {
) -> Option<Decline> {
unsupported_param(
self.supported_openai_param_mappings(),
self.config_params(),
@ -111,13 +102,13 @@ pub fn unsupported_param(
supported: &'static [(&'static str, &'static str)],
config: &'static [&'static str],
optional_params: &Map<String, Value>,
) -> Option<Unsupported> {
) -> Option<Decline> {
if optional_params
.get(STREAM_PARAM)
.and_then(Value::as_bool)
.unwrap_or(false)
{
return Some(Unsupported("streaming"));
return Some(Decline::new("streaming"));
}
optional_params
.keys()
@ -128,29 +119,29 @@ pub fn unsupported_param(
.any(|(_, provider_name)| *provider_name == key)
&& !config.contains(&key.as_str())
})
.then_some(Unsupported("unrecognized request parameter"))
.then_some(Decline::new("unrecognized request parameter"))
}
/// Message shapes the core can translate faithfully: text content, either a
/// plain string or a non-empty list of parts that are all
/// `{"type": "text", "text": ...}`. Tool calls, tool results, and multimodal
/// parts decline so Python's fuller translation handles them.
pub fn unsupported_message(message: &ChatMessage) -> Option<Unsupported> {
pub fn unsupported_message(message: &ChatMessage) -> Option<Decline> {
if message
.extra
.keys()
.any(|key| !IGNORABLE_MESSAGE_FIELDS.contains(&key.as_str()))
{
return Some(Unsupported("unrecognized message field"));
return Some(Decline::new("unrecognized message field"));
}
if !matches!(message.role.as_str(), "system" | "user" | "assistant") {
return Some(Unsupported("unrecognized message role"));
return Some(Decline::new("unrecognized message role"));
}
match &message.content {
None => Some(Unsupported("message without content")),
None => Some(Decline::new("message without content")),
Some(ChatMessageContent::Text(_)) => None,
Some(ChatMessageContent::Parts(parts)) if parts.is_empty() => {
Some(Unsupported("message without content"))
Some(Decline::new("message without content"))
}
Some(ChatMessageContent::Parts(parts)) => parts
.iter()
@ -159,6 +150,6 @@ pub fn unsupported_message(message: &ChatMessage) -> Option<Unsupported> {
|| part.get("text").and_then(Value::as_str).is_none()
|| part.as_object().is_some_and(|object| object.len() != 2)
})
.then_some(Unsupported("non-text message content")),
.then_some(Decline::new("non-text message content")),
}
}

2
litellm-rust/crates/llms/src/base_llm/ocr/error.rs
View file

@ -72,8 +72,6 @@ pub enum Error {
PollOrigin,
#[error("OCR polling timed out")]
PollTimeout,
#[error("unsupported by the rust path: {0}")]
Unsupported(&'static str),
#[error("invalid provider: {0}")]
InvalidProvider(String),
#[error("invalid model: {provider} has no model {model:?} - use one of: {}", supported.join(", "))]

43
litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs
View file

@ -5,6 +5,7 @@ use litellm_auth_aws::{
constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE},
resolve_bedrock_region,
};
use litellm_core_utils::decline::Decline;
use litellm_core_utils::{
core_helpers::{finish_reason_for, unix_now, usage_from_parts},
prompt_templates::factory::{Conversation, TurnRole, build_conversation},
@ -26,7 +27,7 @@ use crate::{
streaming::StreamShape,
transformation::{
BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData,
Unsupported, ValidatedEnvironment, unsupported_message, unsupported_param,
ValidatedEnvironment, unsupported_message, unsupported_param,
},
},
},
@ -169,10 +170,10 @@ impl BaseConfig for AmazonConverseConfig {
let response: ConverseResponse = serde_json::from_value(body).map_err(|error| {
Error::InvalidResponse(format!("invalid Converse response: {error}").into())
})?;
// The route declines tool requests, so anything other than a text block
// is something this path never asked for. Decline; the host falls back.
if response.message_content_is_non_text() {
return Err(Error::Unsupported("non-text response content block"));
return Err(Error::InvalidResponse(
"non-text response content block".into(),
));
}
let text = response.content_text();
let computed = usage_from_parts(
@ -269,7 +270,7 @@ impl BaseConfig for AmazonConverseConfig {
&self,
messages: &[ChatMessage],
optional_params: &Map<String, Value>,
) -> Option<Unsupported> {
) -> Option<Decline> {
unsupported_param(
self.supported_openai_param_mappings(),
CONFIG_PARAMS,
@ -283,7 +284,7 @@ impl BaseConfig for AmazonConverseConfig {
messages
.iter()
.any(has_blank_text)
.then_some(Unsupported("blank message text"))
.then_some(Decline::new("blank message text"))
})
// Converse has no assistant prefill: Python inserts a continue turn
// when a conversation opens or closes on an assistant message, and
@ -296,7 +297,7 @@ impl BaseConfig for AmazonConverseConfig {
.turns
.last()
.is_some_and(|turn| turn.role == TurnRole::Assistant);
(!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Unsupported(
(!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Decline::new(
"conversation does not run user turn to user turn",
))
})
@ -358,11 +359,12 @@ mod tests {
Error,
base_llm::{
auth::AuthScheme,
chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported},
chat::transformation::{BaseConfig, ProviderChatResponseData},
},
bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG,
};
use litellm_auth::CredentialPlacement;
use litellm_core_utils::decline::Decline;
use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage};
use rstest::rstest;
use serde_json::{Map, Value, json};
@ -396,7 +398,7 @@ mod tests {
)
}
fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
fn reason(msgs: Value, opts: Value) -> Option<Decline> {
BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), &params(opts))
}
@ -472,7 +474,7 @@ mod tests {
json!([{"role": "user", "content": "hi"}]),
json!({"stream": true})
),
Some(Unsupported("streaming"))
Some(Decline::new("streaming"))
);
}
@ -483,7 +485,7 @@ mod tests {
json!([{"role": "user", "content": "hi"}]),
json!({"topK": 40})
),
Some(Unsupported("unrecognized request parameter"))
Some(Decline::new("unrecognized request parameter"))
);
}
@ -497,7 +499,7 @@ mod tests {
fn declines_tools_and_other_params_outside_the_allowlist(#[case] param: Value) {
assert_eq!(
reason(json!([{"role": "user", "content": "hi"}]), param.clone()),
Some(Unsupported("unrecognized request parameter")),
Some(Decline::new("unrecognized request parameter")),
"expected {param} to decline"
);
}
@ -514,7 +516,7 @@ mod tests {
json!([{"role": "user", "content": content}, {"role": "user", "content": "hi"}]),
json!({})
),
Some(Unsupported("blank message text")),
Some(Decline::new("blank message text")),
"expected blank content {content} to decline"
);
}
@ -526,7 +528,7 @@ mod tests {
// host before the call rather than an error after it.
assert_eq!(
reason(json!([{"role": "user", "content": []}]), json!({})),
Some(Unsupported("message without content"))
Some(Decline::new("message without content"))
);
assert_eq!(
reason(
@ -547,7 +549,7 @@ mod tests {
]),
json!({})
),
Some(Unsupported(
Some(Decline::new(
"conversation does not run user turn to user turn"
))
);
@ -559,7 +561,7 @@ mod tests {
]),
json!({})
),
Some(Unsupported(
Some(Decline::new(
"conversation does not run user turn to user turn"
))
);
@ -819,7 +821,7 @@ mod tests {
]}]),
json!({})
),
Some(Unsupported("non-text message content"))
Some(Decline::new("non-text message content"))
);
}
@ -846,7 +848,7 @@ mod tests {
}
#[test]
fn declines_a_response_carrying_a_tool_use_block() {
fn a_response_carrying_a_tool_use_block_is_invalid() {
let err = transform_response(json!({
"output": {"message": {"content": [
{"toolUse": {"toolUseId": "t1", "name": "f", "input": {}}}
@ -855,7 +857,10 @@ mod tests {
"usage": {"inputTokens": 1, "outputTokens": 1}
}))
.expect_err("tool use block");
assert_eq!(err, Error::Unsupported("non-text response content block"));
assert_eq!(
err,
Error::InvalidResponse("non-text response content block".into())
);
}
#[test]

9
litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs
View file

@ -1,6 +1,7 @@
use base64::Engine;
use bytes::Buf;
use futures_util::{Stream, StreamExt};
use litellm_core_utils::decline::Decline;
use litellm_framer::{
aws_event_stream::{AwsEventStreamCodec, Message},
frames,
@ -85,10 +86,12 @@ pub fn invoke_chat_stream(invoke_provider: &str, shape: StreamShape) -> Result<C
invoke_anthropic_event_stream,
ModelResponseIterator::new(shape),
)),
InvokeProvider::DeepseekR1 | InvokeProvider::Moonshot => Err(Error::Unsupported(
"Bedrock invoke streaming for this model family",
InvokeProvider::DeepseekR1 | InvokeProvider::Moonshot => Err(Error::Declined(
Decline::new("Bedrock invoke streaming for this model family"),
)),
InvokeProvider::Unsupported => Err(Error::Unsupported("Bedrock invoke streaming")),
InvokeProvider::Unsupported => {
Err(Error::Declined(Decline::new("Bedrock invoke streaming")))
}
}
}

5
litellm-rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude3_transformation.rs
View file

@ -1,3 +1,4 @@
use litellm_core_utils::decline::Decline;
use std::convert::Infallible;
use crate::{
@ -130,9 +131,9 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
_request: MessagesRequest,
_context: &MessagesTransformContext,
) -> Result<MessagesRequest, Error> {
Err(Error::Unsupported(
Err(Error::Declined(Decline::new(
"Bedrock invoke messages request shaping",
))
)))
}
fn secret_names(&self) -> &'static [&'static str] {

6
litellm-rust/crates/llms/src/error.rs
View file

@ -1,3 +1,5 @@
use litellm_core_utils::decline::Decline;
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
pub enum Error {
#[error("expected {expected}, got {actual}")]
@ -11,8 +13,8 @@ pub enum Error {
InvalidRequest(#[source] ErrorDetail),
#[error("invalid response: {0}")]
InvalidResponse(#[source] ErrorDetail),
#[error("unsupported: {0}")]
Unsupported(&'static str),
#[error(transparent)]
Declined(Decline),
#[error(transparent)]
Auth(#[from] litellm_auth::Error),
}

29
litellm-rust/crates/llms/src/openai_like/chat/transformation.rs
View file

@ -243,14 +243,14 @@ fn normalize_choice(position: usize, choice: &Value) -> Result<ChatCompletionsCh
// normalized type cannot carry tool calls at all. Declining is
// terminal at this point, but passing back an empty assistant turn
// would fabricate the reply.
return Err(Error::Unsupported("tool call response"));
return Err(Error::InvalidResponse("tool call response".into()));
}
if message.get("refusal").is_some_and(|value| !value.is_null()) {
return Err(Error::Unsupported("refusal response"));
return Err(Error::InvalidResponse("refusal response".into()));
}
let content = message.get("content");
if content.is_some_and(|value| !value.is_null() && !value.is_string()) {
return Err(Error::Unsupported("non-text response content"));
return Err(Error::InvalidResponse("non-text response content".into()));
}
Ok(ChatCompletionsChoice {
index: choice
@ -279,10 +279,11 @@ mod tests {
Error,
base_llm::{
auth::AuthScheme,
chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported},
chat::transformation::{BaseConfig, ProviderChatResponseData},
},
openai_like::chat::transformation::OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG,
};
use litellm_core_utils::decline::Decline;
use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage};
use rstest::rstest;
use serde_json::{Map, Value, json};
@ -318,7 +319,7 @@ mod tests {
.transform_response("some-model", ProviderChatResponseData { body })
}
fn reason(msgs: Value, opts: Value) -> Option<Unsupported> {
fn reason(msgs: Value, opts: Value) -> Option<Decline> {
OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), &params(opts))
}
@ -528,9 +529,7 @@ mod tests {
}
#[rstest]
fn a_tool_call_response_declines_instead_of_dropping_the_calls() {
// The `json_mode` rewrite needs a request flag the route does not carry, so
// a tool-call answer falls back to Python rather than losing the calls.
fn a_tool_call_response_fails_instead_of_dropping_the_calls() {
assert_eq!(
transform_response(json!({
"model": "m",
@ -547,12 +546,12 @@ mod tests {
"finish_reason": "tool_calls",
}],
})),
Err(Error::Unsupported("tool call response"))
Err(Error::InvalidResponse("tool call response".into()))
);
}
#[rstest]
fn a_refusal_declines_instead_of_returning_an_empty_reply() {
fn a_refusal_fails_instead_of_returning_an_empty_reply() {
assert_eq!(
transform_response(json!({
"model": "m",
@ -561,12 +560,12 @@ mod tests {
"finish_reason": "stop",
}],
})),
Err(Error::Unsupported("refusal response"))
Err(Error::InvalidResponse("refusal response".into()))
);
}
#[rstest]
fn a_non_text_response_content_declines() {
fn a_non_text_response_content_is_invalid() {
assert_eq!(
transform_response(json!({
"model": "m",
@ -575,7 +574,7 @@ mod tests {
"finish_reason": "stop",
}],
})),
Err(Error::Unsupported("non-text response content"))
Err(Error::InvalidResponse("non-text response content".into()))
);
}
@ -585,7 +584,7 @@ mod tests {
fn declines(#[case] opts: Value, #[case] expected: &'static str) {
assert_eq!(
reason(json!([{"role": "user", "content": "hi"}]), opts),
Some(Unsupported(expected))
Some(Decline::new(expected))
);
}
@ -615,7 +614,7 @@ mod tests {
json!([{"role": "user", "content": "hi"}]),
json!({"tools": [{"type": "function", "function": {"name": "f"}}]}),
),
Some(Unsupported("unrecognized request parameter"))
Some(Decline::new("unrecognized request parameter"))
);
}
}

34
litellm-rust/crates/python-bridge/src/errors.rs
View file

@ -20,6 +20,9 @@ pyo3::create_exception!(
);
pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr {
if let Some(declined) = decline_to_pyerr(&error) {
return declined;
}
match error {
Error::Transport(TransportError::Http { status, body }) => {
RustUpstreamError::new_err((status, body))
@ -28,6 +31,14 @@ pub(crate) fn route_error_to_pyerr(error: Error) -> PyErr {
}
}
/// A core decline skips the route's public exception mapping, so Python dispatch catches it
/// and runs the legacy path.
pub(crate) fn decline_to_pyerr(error: &Error) -> Option<PyErr> {
error
.declined()
.map(|decline| RustBridgeDeclined::new_err(decline.reason))
}
/// A request the caller got wrong is a `ValueError`; anything else is a `RuntimeError`.
pub(crate) fn by_fault(is_request: bool, message: String) -> PyErr {
if is_request {
@ -39,10 +50,11 @@ pub(crate) fn by_fault(is_request: bool, message: String) -> PyErr {
#[cfg(test)]
mod tests {
use litellm_core_utils::decline::Decline;
use super::*;
#[rstest::rstest]
#[case::unsupported(Error::Unsupported("test capability"), true)]
#[case::invalid_provider(Error::InvalidProvider("unknown".into()), true)]
#[case::invalid_request(Error::InvalidRequest("empty messages".into()), true)]
#[case::connection(TransportError::Connect("unreachable".into()).into(), false)]
@ -58,6 +70,26 @@ mod tests {
});
}
#[test]
fn a_core_decline_hands_the_call_back_to_python() {
Python::initialize();
Python::attach(|py| {
let declined = route_error_to_pyerr(Error::from(litellm_llms::Error::Declined(
Decline::new("unrecognized request parameter"),
)));
assert!(declined.is_instance_of::<RustBridgeDeclined>(py));
assert_eq!(
declined
.value(py)
.getattr("args")
.unwrap()
.extract::<(String,)>()
.unwrap(),
("unrecognized request parameter".into(),)
);
});
}
#[rstest::rstest]
fn transport_status_survives_python_mapping() {
Python::initialize();

5
litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs
View file

@ -1,3 +1,4 @@
use litellm_core_utils::decline::Decline;
use std::convert::Infallible;
use crate::routes::inference::InferenceHost;
@ -44,9 +45,9 @@ impl PythonBinding for ChatCompletionsPythonHost {
.get("stream")
.is_some_and(|value| value == &serde_json::Value::Bool(true))
{
return Err(InvokeError::Native(Error::Unsupported(
return Err(InvokeError::Native(Error::Declined(Decline::new(
"native Python chat_completions streaming",
)));
))));
}
Ok(call)
}

5
litellm-rust/crates/python-bridge/src/routes/inference.rs
View file

@ -20,7 +20,7 @@ use serde_json::{Map, Value};
use super::{NativeCall, RequestView};
use crate::{
errors::{RustUpstreamError, route_error_to_pyerr},
errors::{RustUpstreamError, decline_to_pyerr, route_error_to_pyerr},
marshal::{RouteOptions, optional_timeout, public_response, python_timeout_seconds},
};
@ -120,6 +120,9 @@ impl InferenceHost {
}
pub fn error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
if let Some(declined) = decline_to_pyerr(&error) {
return Ok(declined);
}
if let Error::Secret(source) = &error
&& let Some(original) = crate::secrets::python_error(py, source.source_error())
{

5
litellm-rust/crates/python-bridge/src/routes/messages/host.rs
View file

@ -20,7 +20,7 @@ use pyo3::{
use serde_json::{Map, Value};
use crate::{
errors::{RustUpstreamError, route_error_to_pyerr},
errors::{RustUpstreamError, decline_to_pyerr, route_error_to_pyerr},
marshal::{optional_timeout, project_optional_fields, public_response, python_timeout_seconds},
python_settings::missing_module,
routes::RequestView,
@ -276,6 +276,9 @@ impl PythonBinding for MessagesPythonHost {
}
fn map_error(&self, py: Python<'_>, error: Error) -> PyResult<PyErr> {
if let Some(declined) = decline_to_pyerr(&error) {
return Ok(declined);
}
if let Error::Secret(source) = &error
&& let Some(original) = crate::secrets::python_error(py, source.source_error())
{

5
litellm-rust/crates/python-bridge/src/routes/responses/host.rs
View file

@ -1,3 +1,4 @@
use litellm_core_utils::decline::Decline;
use std::convert::Infallible;
use crate::routes::inference::InferenceHost;
@ -118,9 +119,9 @@ impl PythonBinding for ResponsesPythonHost {
.get("stream")
.is_some_and(|value| value == &serde_json::Value::Bool(true))
{
return Err(InvokeError::Native(Error::Unsupported(
return Err(InvokeError::Native(Error::Declined(Decline::new(
"native Python responses streaming",
)));
))));
}
Ok(call)
}