scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-09-21 00:21:49 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 74

feat(rust): preserve provider request extensions across Python boundaries

Browse source
This commit is contained in:
Yujong Lee 2026-09-15 13:12:40 -07:00
parent d3929287fe
commit cee13d5d70
28 changed files with 742 additions and 127 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
litellm-rust/crates/core/src/audio_transcription/transformation.rs
View file

@ -17,14 +17,7 @@ pub trait AudioTranscriptionProviderConfig: Sync {
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
fn map_transcription_params(&self, params: &Map<String, Value>) -> Map<String, Value> {
params
.iter()
.filter(|(key, _)| {
self.supported_transcription_params()
.contains(&key.as_str())
})
.map(|(key, value)| (key.clone(), value.clone()))
.collect()
crate::params::provider_fields(params).into()
}
fn transform_transcription_request(

1
litellm-rust/crates/core/src/chat_completions/prepare.rs
View file

@ -40,6 +40,7 @@ pub(super) fn parse_messages(messages: Value) -> Result<Vec<ChatMessage>, Error>
pub(super) fn resolve_request(
request: ChatCompletionsRequest<'_>,
) -> Result<ResolvedChatCompletionsRequest<'_>, Error> {
crate::params::body_overrides(&request.optional_params)?;
let (model, config) = resolve_provider_config(request.model, request.custom_llm_provider)?;
let messages = parse_messages(request.messages)?;
if messages.is_empty() {

58
litellm-rust/crates/core/src/chat_completions/tests.rs
View file

@ -56,6 +56,64 @@ fn resolves_the_provider_from_the_model_prefix() {
assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5"));
}
#[test]
fn extensions_survive_admission_and_provider_mapping() {
for model in [
"anthropic/claude-sonnet-4-5",
"bedrock/anthropic.claude-sonnet-4-5-v1:0",
] {
let prepared = prepare_chat_completions_call(request(
model,
None,
json!([{"role":"user", "content":"hi"}]),
json!({"future":{"nested":[null,false,0]}, "settings":{"a":1},
"extra_body":{"settings":{"b":2}}, "aws_secret_access_key":"not-body"}),
))
.unwrap();
assert_eq!(prepared.body["future"], json!({"nested":[null,false,0]}));
assert_eq!(prepared.body["settings"], json!({"b":2}));
assert!(prepared.body.get("extra_body").is_none());
assert!(prepared.body.get("aws_secret_access_key").is_none());
}
}
#[test]
fn overrides_cannot_bypass_chat_capability_checks() {
for model in ["anthropic/model", "bedrock/model"] {
for fields in [
json!({"stream":true}),
json!({"tools":[]}),
json!({"top_k":10}),
json!({"messages":[]}),
] {
assert!(matches!(
decline(request(
model,
None,
json!([{"role":"user", "content":"hi"}]),
json!({"extra_body":fields})
)),
Error::Unsupported(_)
));
}
}
}
#[test]
fn invalid_overrides_are_terminal_request_errors() {
for value in [json!(false), json!(1), json!([]), json!("invalid")] {
assert!(matches!(
decline(request(
"anthropic/model",
None,
json!([{"role":"user", "content":"hi"}]),
json!({"extra_body":value})
)),
Error::InvalidRequest(_)
));
}
}
#[test]
fn strips_an_explicit_provider_prefix_from_the_model() {
let prepared = prepare_chat_completions_call(request(

51
litellm-rust/crates/core/src/chat_completions/transformation.rs
View file

@ -19,9 +19,8 @@ pub enum ChatCompletionsAuth {
///
/// The core declines rather than guessing: the host turns this into a
/// transparent fallback to the Python implementation, which covers the full
/// surface. Acceptance is an allowlist, so a parameter or message shape the
/// core has never seen declines by construction instead of being translated
/// wrong.
/// surface. Known features needing translation decline; opaque provider body
/// extensions do not imply support for a new message or response shape.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Unsupported(pub &'static str);
@ -103,8 +102,12 @@ pub fn unsupported_param(
config: &'static [&'static str],
optional_params: &Map<String, Value>,
) -> Option<Unsupported> {
if optional_params
.get(STREAM_PARAM)
let overrides = crate::params::body_overrides(optional_params)
.ok()
.flatten();
if overrides
.and_then(|fields| fields.get(STREAM_PARAM))
.or_else(|| optional_params.get(STREAM_PARAM))
.and_then(Value::as_bool)
.unwrap_or(false)
{
@ -112,12 +115,50 @@ pub fn unsupported_param(
}
optional_params
.keys()
.chain(overrides.into_iter().flat_map(|fields| fields.keys()))
.any(|key| {
key != STREAM_PARAM
&& !supported
.iter()
.any(|(_, provider_name)| *provider_name == key)
&& !config.contains(&key.as_str())
&& matches!(
key.as_str(),
"tools"
| "tool_choice"
| "toolConfig"
| "thinking"
| "system"
| "messages"
| "metadata"
| "output_config"
| "outputConfig"
| "requestMetadata"
| "_parallel_tool_use_config"
| "top_k"
| "topK"
| "model"
| "functions"
| "function_call"
| "parallel_tool_calls"
| "response_format"
| "n"
| "logprobs"
| "top_logprobs"
| "modalities"
| "audio"
| "prediction"
| "reasoning_effort"
| "max_completion_tokens"
| "stop"
| "max_tokens"
| "top_p"
| "frequency_penalty"
| "presence_penalty"
| "logit_bias"
| "seed"
| "stream_options"
)
})
.then_some(Unsupported("unrecognized request parameter"))
}

1
litellm-rust/crates/core/src/lib.rs
View file

@ -11,6 +11,7 @@ pub mod messages;
#[cfg(any(feature = "observability", test))]
pub mod observability;
pub mod ocr;
pub mod params;
pub mod providers;
pub mod realtime;
pub mod responses;

12
litellm-rust/crates/core/src/messages/tests.rs
View file

@ -60,6 +60,18 @@ fn provider_config_resolves_anthropic_and_azure_ai() {
assert!(messages_provider_config("openai").is_none());
}
#[test]
fn native_message_extensions_survive_at_each_open_schema_boundary() {
let body = json!({"model":"claude", "messages":[{"role":"user",
"content":[{"type":"text", "text":"hi", "future_block":null,
"cache_control":{"type":"ephemeral", "future_cache":false}}], "future_message":0}],
"future_request":{"extra_body":{"timeout":null}}});
let config = messages_provider_config("anthropic").unwrap();
let request = serde_json::from_value(body.clone()).unwrap();
let transformed = config.transform_request(request).unwrap();
assert_eq!(serde_json::to_value(transformed).unwrap(), body);
}
#[test]
fn truncate_error_body_caps_long_payloads() {
let body = "x".repeat(400);

11
litellm-rust/crates/core/src/messages/types.rs
View file

@ -1,5 +1,6 @@
use std::time::Duration;
use crate::params::OpaqueFields;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
@ -44,7 +45,7 @@ pub struct ContentBlock {
#[serde(skip_serializing_if = "Option::is_none")]
pub cache_control: Option<CacheControl>,
#[serde(flatten)]
pub extra: Map<String, Value>,
pub extra: OpaqueFields,
}
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
@ -56,7 +57,7 @@ pub struct CacheControl {
#[serde(skip_serializing_if = "Option::is_none")]
pub scope: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
pub extra: OpaqueFields,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -64,7 +65,7 @@ pub struct AnthropicMessage {
pub role: String,
pub content: MessageContent,
#[serde(flatten)]
pub extra: Map<String, Value>,
pub extra: OpaqueFields,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -110,7 +111,7 @@ pub struct AnthropicMessagesRequest {
#[serde(skip_serializing_if = "Option::is_none")]
pub inference_geo: Option<String>,
#[serde(flatten)]
pub extra: Map<String, Value>,
pub extra: OpaqueFields,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
@ -130,5 +131,5 @@ pub struct AnthropicMessagesResponse {
#[serde(skip_serializing_if = "Option::is_none")]
pub container: Option<Value>,
#[serde(flatten)]
pub extra: Map<String, Value>,
pub extra: OpaqueFields,
}

116
litellm-rust/crates/core/src/ocr/prepare.rs
View file

@ -1,23 +1,18 @@
use serde::{Deserialize, Serialize, de::DeserializeOwned};
use serde_json::{Map, Value};
use serde::{Serialize, de::DeserializeOwned};
use serde_json::Value;
use super::OcrClient;
use super::error::{OcrError, OcrRequestError};
use super::hooks::OcrDuringCallRequest;
use super::types::{LiteLLMOcrRequest, OcrDocument};
#[derive(Debug, Deserialize)]
pub(crate) struct ParsedProviderParams<T> {
#[serde(flatten)]
pub known: T,
#[serde(default, flatten)]
pub extra_params: Map<String, Value>,
}
pub(crate) use crate::params::ParsedProviderParams;
#[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)]
pub(crate) fn _prepare_ocr_request<T: DeserializeOwned>(
request: &LiteLLMOcrRequest,
) -> Result<ParsedProviderParams<T>, OcrRequestError> {
crate::params::body_overrides(&request.optional_params).map_err(parameter_error)?;
super::wire::decode_request_value(
Value::Object(request.optional_params.clone()),
"optional_params",
@ -26,35 +21,19 @@ pub(crate) fn _prepare_ocr_request<T: DeserializeOwned>(
pub(crate) fn merge_extra_params<B: Serialize>(
body: &B,
extra_params: Map<String, Value>,
extra_params: crate::params::OpaqueFields,
) -> Result<Value, OcrRequestError> {
let Value::Object(fields) =
serde_json::to_value(body).map_err(|_| OcrRequestError::RequestField {
path: "body".into(),
})?
else {
return Err(OcrRequestError::RequestField {
path: "body".into(),
});
};
let extra_body = extra_params
.get("extra_body")
.and_then(Value::as_object)
.cloned()
.unwrap_or_default()
.into_iter()
.collect::<Map<String, Value>>();
Ok(Value::Object(
fields
.into_iter()
.chain(
extra_params
.into_iter()
.filter(|(name, _)| name != "extra_body"),
)
.chain(extra_body)
.collect(),
))
crate::params::compose_body(body, &extra_params, &[]).map_err(parameter_error)
}
fn parameter_error(error: crate::params::Error) -> OcrRequestError {
OcrRequestError::RequestField {
path: match error {
crate::params::Error::ExtraBody => "extra_body",
crate::params::Error::Body => "body",
}
.into(),
}
}
pub(crate) async fn transform_request_body<B>(
@ -69,16 +48,26 @@ pub(crate) async fn transform_request_body<B>(
where
B: Serialize + DeserializeOwned,
{
let consumed = super::wire::consumed_optional_param_names(
&request.model,
Some(request.adapter.provider().as_str()),
)?;
let body = crate::params::compose_body(&body, &request.optional_params, &consumed)
.map_err(parameter_error)?;
let (body, headers) = if request.hooks.intercepts_requests() {
let body = serde_json::to_value(body).map_err(|_| OcrRequestError::RequestField {
path: "body".into(),
})?;
let overrides =
crate::params::body_overrides(&request.optional_params).map_err(parameter_error)?;
let retained_fields = request
.optional_params
.keys()
.filter(|name| body.get(*name).is_some())
.filter(|name| !overrides.is_some_and(|fields| fields.contains_key(*name)))
.cloned()
.chain(retains_document.then(|| "document".to_string()))
.chain(
(retains_document
&& !overrides.is_some_and(|fields| fields.contains_key("document")))
.then(|| "document".to_string()),
)
.collect();
let changed = request
.hooks
@ -91,17 +80,14 @@ where
retained_fields,
})
.await?;
let body = OcrWireBody::<B>::decode(changed.body)?;
validate(&body.body)?;
(body, changed.headers)
let projected: B =
super::wire::decode_request_value(changed.body.clone(), "guardrail.body")?;
validate(&projected)?;
(changed.body, changed.headers)
} else {
(
OcrWireBody {
body,
extra: Map::new(),
},
headers.to_vec(),
)
let projected: B = super::wire::decode_request_value(body.clone(), "body")?;
validate(&projected)?;
(body, headers.to_vec())
};
build_http_request(client, request, url, &headers, &body)
}
@ -151,38 +137,12 @@ pub(crate) async fn guardrail_document(
Ok((document, changed.headers))
}
#[derive(Serialize)]
struct OcrWireBody<B> {
#[serde(flatten)]
body: B,
#[serde(flatten)]
extra: Map<String, Value>,
}
impl<B: Serialize + DeserializeOwned> OcrWireBody<B> {
fn decode(value: Value) -> Result<Self, OcrRequestError> {
let body: B = super::wire::decode_request_value(value.clone(), "guardrail.body")?;
let Value::Object(fields) = value else {
return Err(OcrRequestError::RequestField {
path: "guardrail.body".into(),
});
};
let known = serde_json::to_value(&body).map_err(|_| OcrRequestError::RequestField {
path: "guardrail.body".into(),
})?;
let extra = fields
.into_iter()
.filter(|(key, _)| known.get(key).is_none())
.collect();
Ok(Self { body, extra })
}
}
pub(crate) fn credential_env(name: &str) -> Option<String> {
std::env::var(name).ok()
}
#[cfg(test)]
mod tests {
use serde::Deserialize;
use serde_json::json;
use super::*;

270
litellm-rust/crates/core/src/params.rs Normal file
View file

@ -0,0 +1,270 @@
use std::ops::{Deref, DerefMut};
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)]
#[serde(transparent)]
pub struct OpaqueFields(Map<String, Value>);
impl Deref for OpaqueFields {
type Target = Map<String, Value>;
fn deref(&self) -> &Self::Target {
&self.0
}
}
impl DerefMut for OpaqueFields {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.0
}
}
impl From<Map<String, Value>> for OpaqueFields {
fn from(fields: Map<String, Value>) -> Self {
Self(fields)
}
}
impl From<OpaqueFields> for Map<String, Value> {
fn from(fields: OpaqueFields) -> Self {
fields.0
}
}
impl FromIterator<(String, Value)> for OpaqueFields {
fn from_iter<T: IntoIterator<Item = (String, Value)>>(fields: T) -> Self {
Self(fields.into_iter().collect())
}
}
#[derive(Debug, Deserialize)]
pub(crate) struct ParsedProviderParams<T> {
#[serde(flatten)]
pub known: T,
#[serde(default, flatten)]
pub extra_params: OpaqueFields,
}
#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)]
pub enum Error {
#[error("extra_body must be an object")]
ExtraBody,
#[error("body must be a JSON object")]
Body,
}
impl From<Error> for crate::Error {
fn from(error: Error) -> Self {
Self::InvalidRequest(error.to_string())
}
}
pub fn is_control_param(name: &str) -> bool {
matches!(
name,
"api_key"
| "api_base"
| "custom_llm_provider"
| "extra_headers"
| "extra_query"
| "timeout"
| "timeout_seconds"
| "request_timeout"
| "max_retries"
| "req_format"
| "max_response_bytes"
| "input_sources"
| "litellm_call_id"
| "litellm_logging_obj"
| "litellm_metadata"
| "proxy_server_request"
| "callbacks"
| "success_callback"
| "failure_callback"
| "guardrails"
| "drop_params"
| "additional_drop_params"
| "allowed_openai_params"
| "azure_ad_token"
| "azure_ad_token_provider"
| "tenant_id"
| "client_id"
| "client_secret"
| "azure_scope"
| "azure_authority_host"
| "azure_credential"
| "azure_federated_token_file"
| "enable_azure_ad_token_refresh"
| "vertex_credentials"
| "vertex_ai_credentials"
| "vertex_project"
| "vertex_ai_project"
| "vertex_location"
| "vertex_ai_location"
| "aws_access_key_id"
| "aws_secret_access_key"
| "aws_session_token"
| "aws_region_name"
| "aws_session_name"
| "aws_profile_name"
| "aws_role_name"
| "aws_web_identity_token"
| "aws_sts_endpoint"
| "aws_external_id"
| "aws_bedrock_runtime_endpoint"
)
}
pub fn body_overrides(params: &Map<String, Value>) -> Result<Option<&Map<String, Value>>, Error> {
match params.get("extra_body") {
None | Some(Value::Null) => Ok(None),
Some(Value::Object(fields)) => Ok(Some(fields)),
Some(_) => Err(Error::ExtraBody),
}
}
pub fn provider_fields(params: &Map<String, Value>) -> OpaqueFields {
params
.iter()
.filter(|(name, _)| !is_control_param(name))
.map(|(name, value)| (name.clone(), value.clone()))
.collect()
}
pub fn compose_body<B: Serialize>(
body: &B,
params: &Map<String, Value>,
consumed: &[&str],
) -> Result<Value, Error> {
let overrides = body_overrides(params)?;
let Value::Object(fields) = serde_json::to_value(body).map_err(|_| Error::Body)? else {
return Err(Error::Body);
};
Ok(Value::Object(
fields
.into_iter()
.chain(
params
.iter()
.filter(|(name, _)| !consumed.contains(&name.as_str()))
.chain(overrides.into_iter().flat_map(|fields| fields.iter()))
.filter(|(name, _)| name.as_str() != "extra_body" && !is_control_param(name))
.map(|(name, value)| (name.clone(), value.clone())),
)
.collect(),
))
}
#[cfg(test)]
mod tests {
use super::*;
use rstest::rstest;
use serde_json::json;
#[rstest]
#[case::null(json!(null))]
#[case::false_value(json!(false))]
#[case::zero(json!(0))]
#[case::empty_string(json!(""))]
#[case::empty_array(json!([]))]
#[case::empty_object(json!({}))]
#[case::nested_names(json!({"timeout":null,"extra_body":{"api_key":"data"}}))]
fn unknown_values_survive_composition(#[case] value: Value) {
let params = json!({"future":value});
assert_eq!(
compose_body(&json!({}), params.as_object().unwrap(), &[]).unwrap(),
params
);
}
#[test]
fn opaque_fields_do_not_apply_request_policy() {
let value =
json!({"extra_body":{"api_key":"data"},"timeout":null,"future":[false,0,"",[],{}]});
let fields: OpaqueFields = serde_json::from_value(value.clone()).unwrap();
assert_eq!(serde_json::to_value(fields).unwrap(), value);
}
#[test]
fn overrides_replace_objects_without_dropping_nulls_or_nested_names() {
let params = json!({
"future":{"extra_body":{"timeout":null}},
"api_key":"secret",
"extra_body":{"settings":{"b":2},"explicit_null":null,"aws_secret_access_key":"secret"}
});
assert_eq!(
compose_body(
&json!({"settings":{"a":1}}),
params.as_object().unwrap(),
&[]
)
.unwrap(),
json!({"settings":{"b":2},"explicit_null":null,"future":{"extra_body":{"timeout":null}}})
);
}
#[test]
fn consumed_options_are_not_remapped_but_overrides_are_applied() {
let params = json!({"temperature":0.2,"future":true,"extra_body":{"inferenceConfig":{"temperature":0.7}}});
assert_eq!(
compose_body(
&json!({"inferenceConfig":{"temperature":0.2,"maxTokens":10}}),
params.as_object().unwrap(),
&["temperature"]
)
.unwrap(),
json!({"inferenceConfig":{"temperature":0.7},"future":true})
);
}
#[rstest]
#[case::absent(json!({}))]
#[case::null(json!({"extra_body":null}))]
#[case::empty(json!({"extra_body":{}}))]
fn empty_overrides(#[case] params: Value) {
assert_eq!(
compose_body(&json!({}), params.as_object().unwrap(), &[]).unwrap(),
json!({})
);
}
#[rstest]
#[case::boolean(json!(false))]
#[case::number(json!(1))]
#[case::array(json!([]))]
#[case::string(json!("body"))]
fn invalid_overrides(#[case] value: Value) {
let params = json!({"extra_body":value});
assert_eq!(
compose_body(&json!({}), params.as_object().unwrap(), &[]),
Err(Error::ExtraBody)
);
}
#[rstest]
fn controls_are_not_body_extensions(
#[values(
"api_key",
"timeout",
"callbacks",
"aws_secret_access_key",
"vertex_credentials",
"azure_ad_token"
)]
name: &str,
#[values(false, true)] explicit_override: bool,
) {
let fields = Map::from_iter([(name.into(), json!("secret-or-control"))]);
let params = if explicit_override {
json!({"extra_body":fields})
} else {
Value::Object(fields)
};
assert_eq!(
compose_body(&json!({}), params.as_object().unwrap(), &[]).unwrap(),
json!({})
);
}
}

2
litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs
View file

@ -160,7 +160,7 @@ fn accepts_an_explicit_stream_false() {
}
#[test]
fn declines_any_param_outside_the_allowlist() {
fn declines_known_features_requiring_python_translation() {
for param in [
json!({"tools": []}),
json!({"tool_choice": {"type": "auto"}}),

6
litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs
View file

@ -146,7 +146,11 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig {
optional_params: Map<String, Value>,
) -> Result<ProviderChatRequestData, Error> {
Ok(ProviderChatRequestData {
body: anthropic_body(model, &build_conversation(&messages), optional_params),
body: crate::params::compose_body(
&anthropic_body(model, &build_conversation(&messages), Map::new()),
&optional_params,
&["stream"],
)?,
})
}

2
litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs
View file

@ -86,7 +86,7 @@ fn text_content_block(text: String) -> ContentBlock {
]);
ContentBlock {
cache_control: None,
extra,
extra: extra.into(),
}
}

56
litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs
View file

@ -71,17 +71,27 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig {
inference_config.insert("temperature".to_string(), temperature.clone());
}
Ok(AudioTranscriptionRequestData {
body: json!({
"messages": [{
"role": "user",
"content": [
{"audio": {"format": format, "source": {"bytes": data}}},
{"text": instruction}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": inference_config,
}),
body: crate::params::compose_body(
&json!({
"messages": [{
"role": "user",
"content": [
{"audio": {"format": format, "source": {"bytes": data}}},
{"text": instruction}
]
}],
"system": [{"text": "You are a transcription assistant."}],
"inferenceConfig": inference_config,
}),
&optional_params,
&[
"language",
"prompt",
"temperature",
"response_format",
"timestamp_granularities",
],
)?,
})
}
@ -197,6 +207,30 @@ mod tests {
assert_eq!(result.into_json(), json!({"text": "hello world"}));
}
#[test]
fn extensions_survive_mapping_and_override_the_final_body() {
let params = json!({"language":"en", "temperature":0.1, "future":null,
"aws_secret_access_key":"secret", "extra_body":{"inferenceConfig":{"temperature":0.7}}});
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.transform_transcription_request(
"model",
json!({"data":"AQI=", "format":"wav"}),
BEDROCK_AUDIO_TRANSCRIPTION_CONFIG
.map_transcription_params(params.as_object().unwrap()),
)
.unwrap();
assert_eq!(result.body["inferenceConfig"], json!({"temperature":0.7}));
assert_eq!(result.body.get("future"), Some(&Value::Null));
for name in [
"language",
"temperature",
"extra_body",
"aws_secret_access_key",
] {
assert!(result.body.get(name).is_none());
}
}
#[test]
fn invalid_audio_is_rejected() {
let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.transform_transcription_request(

2
litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs
View file

@ -123,7 +123,7 @@ fn declines_top_k_because_python_routes_it_by_base_model() {
}
#[test]
fn declines_tools_and_other_params_outside_the_allowlist() {
fn declines_known_features_requiring_python_translation() {
for param in [
json!({"tools": []}),
json!({"tool_choice": {"auto": {}}}),

10
litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs
View file

@ -216,7 +216,15 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig {
optional_params: Map<String, Value>,
) -> Result<ProviderChatRequestData, Error> {
Ok(ProviderChatRequestData {
body: converse_body(&build_conversation(&messages), &optional_params),
body: crate::params::compose_body(
&converse_body(&build_conversation(&messages), &optional_params),
&optional_params,
&SUPPORTED_PARAMS
.iter()
.map(|(_, name)| *name)
.chain(["stream"])
.collect::<Vec<_>>(),
)?,
})
}

17
litellm-rust/crates/core/src/realtime/types.rs
View file

@ -1,5 +1,5 @@
use crate::params::OpaqueFields;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
/// A single realtime event exchanged over the WebSocket.
///
@ -13,7 +13,7 @@ pub struct RealtimeEvent {
#[serde(rename = "type")]
pub event_type: String,
#[serde(flatten)]
pub data: Map<String, Value>,
pub data: OpaqueFields,
}
/// One or more typed events produced by a realtime transform.
@ -34,6 +34,7 @@ impl RealtimeTransformResult {
#[cfg(test)]
mod tests {
use super::*;
use serde_json::Value;
fn event(raw: &str) -> RealtimeEvent {
serde_json::from_str(raw).expect("valid event json")
@ -57,4 +58,16 @@ mod tests {
let result = RealtimeTransformResult::passthrough(parsed.clone());
assert_eq!(result.events, vec![parsed]);
}
#[test]
fn native_events_do_not_interpret_sdk_extension_names() {
let raw = serde_json::json!({"type":"session.update", "extra_body":{"timeout":null}, "future":[false,0]});
let parsed: RealtimeEvent = serde_json::from_value(raw.clone()).unwrap();
let result =
crate::providers::openai::realtime::transformation::transform_realtime_request(
&parsed, "model",
)
.unwrap();
assert_eq!(serde_json::to_value(&result.events[0]).unwrap(), raw);
}
}

5
litellm-rust/crates/core/src/responses/types.rs
View file

@ -1,5 +1,6 @@
use crate::params::OpaqueFields;
use serde::{Deserialize, Deserializer, Serialize, Serializer};
use serde_json::{Map, Value};
use serde_json::Value;
#[derive(Clone, Debug, PartialEq, Eq)]
pub enum ResponsesWsEventType {
@ -58,7 +59,7 @@ pub struct ResponsesWsEvent {
#[serde(rename = "type")]
pub event_type: ResponsesWsEventType,
#[serde(flatten)]
pub data: Map<String, Value>,
pub data: OpaqueFields,
}
impl ResponsesWsEvent {

7
litellm-rust/crates/core/tests/ocr.rs
View file

@ -58,7 +58,8 @@ async fn facade_executes_direct_mistral_once() {
let result = perform_ocr(wire_request(
"mistral/model",
&base,
json!({"pages":"0,2-4","extract_header":true,"unknown":"ignored"}),
json!({"pages":"0,2-4","extract_header":true,"unknown":{"nested":[null,false,0]},
"extra_body":{"future":true,"extract_header":false},"timeout":42}),
))
.await
.unwrap();
@ -80,7 +81,9 @@ async fn facade_executes_direct_mistral_once() {
"model":"model",
"document":{"type":"document_url","document_url":"data:application/pdf;base64,YWJj"},
"pages":"0,2-4",
"extract_header":true
"extract_header":false,
"unknown":{"nested":[null,false,0]},
"future":true
})
);
}

3
litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs
View file

@ -45,7 +45,8 @@ async fn facade_executes_vertex_deepseek_at_the_openai_endpoint() {
let body = request_body(&requests[0]);
assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas");
assert_eq!(body["temperature"], 0.1);
assert!(body.get("future_ocr_option").is_none());
assert_eq!(body["future_ocr_option"], true);
assert_eq!(body["provider_option"], "value");
assert!(body.get("extra_body").is_none());
assert_eq!(
body["messages"][0]["content"][0],

3
litellm-rust/crates/core/tests/vertex_ai_ocr.rs
View file

@ -141,7 +141,8 @@ async fn adapters_build_complete_requests_and_share_mistral_normalization() {
"model": "mistral-ocr-maas",
"document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"},
"pages": [0, 2],
"include_image_base64": true
"include_image_base64": true,
"unknown": "ignored"
})
);
}

10
litellm-rust/crates/python-bridge/src/marshal.rs
View file

@ -92,10 +92,16 @@ pub(crate) fn project_optional_fields(
kwargs: &Bound<'_, PyDict>,
names: &[&str],
) -> PyResult<Map<String, Value>> {
names
let selected: Vec<String> = kwargs
.py()
.import("litellm.rust_bridge.params")?
.getattr("provider_param_names")?
.call1((kwargs, names.to_vec()))?
.extract()?;
selected
.iter()
.filter_map(|name| match kwargs.get_item(name) {
Ok(Some(value)) => Some(from_py(&value).map(|value| ((*name).to_string(), value))),
Ok(Some(value)) => Some(from_py(&value).map(|value| (name.clone(), value))),
Ok(None) => None,
Err(error) => Some(Err(error)),
})

19
litellm/rust_bridge/params.py Normal file
View file

@ -0,0 +1,19 @@
"""Select SDK extension fields without copying or converting their values."""
from collections.abc import Mapping, Sequence
from typing import Final
from litellm.types.utils import all_litellm_params
def provider_param_names(kwargs: Mapping[str, object], consumed: Sequence[str]) -> tuple[str, ...]:
sdk_fields: Final = frozenset(all_litellm_params) | {
"model",
"document",
"timeout",
"extra_headers",
"custom_llm_provider",
"input_sources",
}
consumed_fields: Final = frozenset(consumed)
return tuple(name for name in kwargs if name in consumed_fields or name not in sdk_fields)

24
tests/test_litellm/rust_bridge/test_params.py Normal file
View file

@ -0,0 +1,24 @@
from typing import Final
from litellm.rust_bridge.params import provider_param_names
def test_selects_unknown_fields_and_consumed_controls_without_reading_values() -> None:
callback: Final = object()
future: Final = {"nested": [None, False, 0]}
kwargs: Final = {
"model": "mistral/model",
"litellm_logging_obj": callback,
"metadata": callback,
"vertex_credentials": "credentials",
"future": future,
"extra_body": {"future": None},
}
assert provider_param_names(kwargs, ("vertex_credentials",)) == ("vertex_credentials", "future", "extra_body")
assert kwargs["future"] is future
assert kwargs["litellm_logging_obj"] is callback
def test_route_can_explicitly_consume_a_name_also_used_by_sdk() -> None:
assert provider_param_names({"metadata": {"provider": True}}, ("metadata",)) == ("metadata",)

7
tests/test_litellm_rust/ocr/test_cohere.py
View file

@ -40,7 +40,12 @@ async def test_public_cohere_request_and_normalization(
request: Final = recording_server.requests[0]
assert request.path == ("/providers/cohere/v2/parse" if model.startswith("azure_ai/") else "/v2/parse")
assert request.headers["authorization"] == "Bearer test-key"
assert request.body == {"model": model.split("/", 1)[1], "document": IMAGE, "output_format": "markdown"}
assert request.body == {
"model": model.split("/", 1)[1],
"document": IMAGE,
"output_format": "markdown",
"unrecognized": True,
}
assert [page.index for page in response.pages] == [4, 1]
assert response.pages[0].markdown == "receipt"
assert response.pages[0].images[0].bbox == BOX

2
tests/test_litellm_rust/ocr/test_lifecycle.py
View file

@ -529,7 +529,7 @@ async def test_retained_argument_aliases_and_body_roots_survive_envelope_replace
) -> None:
pages: Final = [0]
document: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}
opaque: Final = object()
opaque: Final = {"future": [None, False, 0]}
observed: Final = []
class Observe(Logging):

114
tests/test_litellm_rust/ocr/test_requests.py
View file

@ -17,6 +17,116 @@ from tests.test_litellm_rust.support.requests import (
pytestmark = pytest.mark.requires_rust_extension
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
@pytest.mark.parametrize("wrapped", [False, True], ids=["kwargs", "extra-body"])
@pytest.mark.parametrize(
"value",
[None, False, 0, "", [], {}, {"timeout": None, "extra_body": {"api_key": "data"}}],
ids=["null", "false", "zero", "empty-string", "empty-array", "empty-object", "nested-names"],
)
async def test_native_ocr_extension_values_cross_the_boundary(
ocr_server: RecordingServer, asynchronous: bool, wrapped: bool, value: object
) -> None:
fields: Final = {"future": value}
arguments: Final = {"extra_body": fields} if wrapped else fields
if asynchronous:
await call_native_aocr(ocr_server, **arguments)
else:
call_native_ocr(ocr_server, **arguments)
assert_native_request(ocr_server)
assert ocr_server.requests[0].body == {
"model": "mistral-ocr-latest",
"document": OCR_DOCUMENT,
"future": value,
}
def test_native_ocr_body_hook_edits_are_not_overwritten_by_extra_body(ocr_server: RecordingServer) -> None:
observed: Final = []
class Edit(RecordingLogger):
def log_pre_api_call(self, model, messages, kwargs):
body = kwargs["additional_args"]["complete_input_dict"]
observed.append(body["future"])
body["future"] = {"from": "hook"}
body.pop("removed")
call_native_ocr(
ocr_server,
future={"from": "kwargs"},
extra_body={"future": {"from": "extra_body"}, "removed": True},
callbacks=[Edit()],
)
assert observed == [{"from": "extra_body"}]
assert ocr_server.requests[0].body["future"] == {"from": "hook"}
assert "removed" not in ocr_server.requests[0].body
def test_native_ocr_document_override_survives_python_hook_projection(ocr_server: RecordingServer) -> None:
document: Final = {"type": "document_url", "document_url": "https://example.com/replacement.pdf"}
call_native_ocr(ocr_server, extra_body={"document": document})
assert_native_request(ocr_server)
assert ocr_server.requests[0].body["document"] == document
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_native_ocr_preserves_extensions_and_applies_shallow_overrides(
ocr_server: RecordingServer, asynchronous: bool
) -> None:
future: Final = {"nested": [None, False, 0, "", [], {}], "timeout": "provider-data"}
overrides: Final = {"settings": {"b": 2}, "extract_header": False, "explicit_null": None}
arguments: Final = {
"future": future,
"settings": {"a": 1},
"extract_header": True,
"extra_body": overrides,
}
if asynchronous:
await call_native_aocr(ocr_server, **arguments)
else:
call_native_ocr(ocr_server, **arguments)
assert_native_request(ocr_server)
assert ocr_server.requests[0].body == {
"model": "mistral-ocr-latest",
"document": OCR_DOCUMENT,
"future": future,
"settings": {"b": 2},
"extract_header": False,
"explicit_null": None,
}
assert arguments["future"] is future
assert arguments["extra_body"] is overrides
assert overrides == {"settings": {"b": 2}, "extract_header": False, "explicit_null": None}
@pytest.mark.parametrize("extra_body", [False, 1, [], "invalid"])
def test_native_ocr_rejects_non_object_overrides_without_sending(
ocr_server: RecordingServer, extra_body: object
) -> None:
ocr_server.expected_requests = 0
with pytest.raises(litellm.BadRequestError, match="extra_body"):
call_native_ocr(ocr_server, extra_body=extra_body)
assert ocr_server.requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_native_ocr_rejects_non_json_provider_options_without_sending(
ocr_server: RecordingServer, asynchronous: bool
) -> None:
ocr_server.expected_requests = 0
if asynchronous:
with pytest.raises(litellm.APIConnectionError, match="unsupported type object"):
await call_native_aocr(ocr_server, future=object())
else:
with pytest.raises(litellm.APIConnectionError, match="unsupported type object"):
call_native_ocr(ocr_server, future=object())
assert ocr_server.requests == []
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_native_azure_ocr_uses_token_provider_result_as_bearer_token(
@ -479,7 +589,7 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen
from litellm.models.credentials import CredentialItem
pages: Final = [0]
opaque: Final = object()
opaque: Final = {"future": [None, False, 0]}
monkeypatch.setenv("MISTRAL_API_KEY", "environment-key")
monkeypatch.setattr(
litellm,
@ -517,6 +627,8 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen
assert ocr_server.requests[0].headers["authorization"] == f"Bearer {expected_key}"
assert ocr_server.requests[0].body["pages"] == [0, 2]
assert ocr_server.requests[0].body["opaque"] == opaque
@pytest.mark.parametrize("source", ["sdk", "proxy"])
@pytest.mark.parametrize(

4
tests/test_litellm_rust/test_ocr.py
View file

@ -122,14 +122,14 @@ def test_native_lifecycle_core_encodes_python_file_input(
document={"type": "file", "file": file_input, "mime_type": mime_type},
api_key="test-key",
api_base=f"http://127.0.0.1:{server.server_port}",
opaque_extension=object(),
opaque_extension={"future": [None, False, 0]},
)
assert response.pages[0].markdown == "native OCR response"
assert requests[0]["body"]["document"] == {
"type": expected_type,
expected_field: expected_uri,
}
assert "opaque_extension" not in requests[0]["body"]
assert requests[0]["body"]["opaque_extension"] == {"future": [None, False, 0]}
@pytest.mark.parametrize("asynchronous", [False, True])

47
tests/test_litellm_rust/test_request_extensions.py Normal file
View file

@ -0,0 +1,47 @@
from typing import Final
import pytest
import litellm
from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec
pytestmark = pytest.mark.requires_rust_extension
@pytest.mark.asyncio
@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"])
async def test_completion_extensions_cross_python_rust_and_http(
recording_server: RecordingServer, asynchronous: bool
) -> None:
recording_server.default_response = ResponseSpec(
body={
"id": "msg_test",
"type": "message",
"role": "assistant",
"model": "claude-sonnet-4-5",
"content": [{"type": "text", "text": "hello"}],
"stop_reason": "end_turn",
"stop_sequence": None,
"usage": {"input_tokens": 2, "output_tokens": 1},
}
)
options: Final = {
"model": "anthropic/claude-sonnet-4-5",
"messages": [{"role": "user", "content": "hi"}],
"api_base": recording_server.base_url,
"api_key": "sk-test",
"max_tokens": 16,
"future_provider_option": {"nested": [None, False, 0]},
"extra_body": {"another_future_option": {"enabled": True}},
}
response: Final = await litellm.acompletion(**options) if asynchronous else litellm.completion(**options)
assert response.choices[0].message.content == "hello"
assert len(recording_server.requests) == 1
request: Final = recording_server.requests[0]
assert not request.headers.get("user-agent", "").startswith("python-httpx")
assert request.body["future_provider_option"] == {"nested": [None, False, 0]}
assert request.body["another_future_option"] == {"enabled": True}
assert request.body["max_tokens"] == 16
assert "extra_body" not in request.body
assert "api_key" not in request.body