diff --git a/litellm-rust/crates/core/src/audio_transcription/transformation.rs b/litellm-rust/crates/core/src/audio_transcription/transformation.rs index aa9846427dc..ad38035e935 100644 --- a/litellm-rust/crates/core/src/audio_transcription/transformation.rs +++ b/litellm-rust/crates/core/src/audio_transcription/transformation.rs @@ -17,14 +17,7 @@ pub trait AudioTranscriptionProviderConfig: Sync { #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] fn map_transcription_params(&self, params: &Map) -> Map { - params - .iter() - .filter(|(key, _)| { - self.supported_transcription_params() - .contains(&key.as_str()) - }) - .map(|(key, value)| (key.clone(), value.clone())) - .collect() + crate::params::provider_fields(params).into() } fn transform_transcription_request( diff --git a/litellm-rust/crates/core/src/chat_completions/prepare.rs b/litellm-rust/crates/core/src/chat_completions/prepare.rs index 3be2ba21de4..4f4f52ce163 100644 --- a/litellm-rust/crates/core/src/chat_completions/prepare.rs +++ b/litellm-rust/crates/core/src/chat_completions/prepare.rs @@ -40,6 +40,7 @@ pub(super) fn parse_messages(messages: Value) -> Result, Error> pub(super) fn resolve_request( request: ChatCompletionsRequest<'_>, ) -> Result, Error> { + crate::params::body_overrides(&request.optional_params)?; let (model, config) = resolve_provider_config(request.model, request.custom_llm_provider)?; let messages = parse_messages(request.messages)?; if messages.is_empty() { diff --git a/litellm-rust/crates/core/src/chat_completions/tests.rs b/litellm-rust/crates/core/src/chat_completions/tests.rs index f8594dee447..410b41f7da5 100644 --- a/litellm-rust/crates/core/src/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/chat_completions/tests.rs @@ -56,6 +56,64 @@ fn resolves_the_provider_from_the_model_prefix() { assert_eq!(prepared.body["model"], json!("claude-sonnet-4-5")); } +#[test] +fn extensions_survive_admission_and_provider_mapping() { + for model in [ + "anthropic/claude-sonnet-4-5", + "bedrock/anthropic.claude-sonnet-4-5-v1:0", + ] { + let prepared = prepare_chat_completions_call(request( + model, + None, + json!([{"role":"user", "content":"hi"}]), + json!({"future":{"nested":[null,false,0]}, "settings":{"a":1}, + "extra_body":{"settings":{"b":2}}, "aws_secret_access_key":"not-body"}), + )) + .unwrap(); + assert_eq!(prepared.body["future"], json!({"nested":[null,false,0]})); + assert_eq!(prepared.body["settings"], json!({"b":2})); + assert!(prepared.body.get("extra_body").is_none()); + assert!(prepared.body.get("aws_secret_access_key").is_none()); + } +} + +#[test] +fn overrides_cannot_bypass_chat_capability_checks() { + for model in ["anthropic/model", "bedrock/model"] { + for fields in [ + json!({"stream":true}), + json!({"tools":[]}), + json!({"top_k":10}), + json!({"messages":[]}), + ] { + assert!(matches!( + decline(request( + model, + None, + json!([{"role":"user", "content":"hi"}]), + json!({"extra_body":fields}) + )), + Error::Unsupported(_) + )); + } + } +} + +#[test] +fn invalid_overrides_are_terminal_request_errors() { + for value in [json!(false), json!(1), json!([]), json!("invalid")] { + assert!(matches!( + decline(request( + "anthropic/model", + None, + json!([{"role":"user", "content":"hi"}]), + json!({"extra_body":value}) + )), + Error::InvalidRequest(_) + )); + } +} + #[test] fn strips_an_explicit_provider_prefix_from_the_model() { let prepared = prepare_chat_completions_call(request( diff --git a/litellm-rust/crates/core/src/chat_completions/transformation.rs b/litellm-rust/crates/core/src/chat_completions/transformation.rs index d7b9704c46c..6bd36fc6047 100644 --- a/litellm-rust/crates/core/src/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/chat_completions/transformation.rs @@ -19,9 +19,8 @@ pub enum ChatCompletionsAuth { /// /// The core declines rather than guessing: the host turns this into a /// transparent fallback to the Python implementation, which covers the full -/// surface. Acceptance is an allowlist, so a parameter or message shape the -/// core has never seen declines by construction instead of being translated -/// wrong. +/// surface. Known features needing translation decline; opaque provider body +/// extensions do not imply support for a new message or response shape. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct Unsupported(pub &'static str); @@ -103,8 +102,12 @@ pub fn unsupported_param( config: &'static [&'static str], optional_params: &Map, ) -> Option { - if optional_params - .get(STREAM_PARAM) + let overrides = crate::params::body_overrides(optional_params) + .ok() + .flatten(); + if overrides + .and_then(|fields| fields.get(STREAM_PARAM)) + .or_else(|| optional_params.get(STREAM_PARAM)) .and_then(Value::as_bool) .unwrap_or(false) { @@ -112,12 +115,50 @@ pub fn unsupported_param( } optional_params .keys() + .chain(overrides.into_iter().flat_map(|fields| fields.keys())) .any(|key| { key != STREAM_PARAM && !supported .iter() .any(|(_, provider_name)| *provider_name == key) && !config.contains(&key.as_str()) + && matches!( + key.as_str(), + "tools" + | "tool_choice" + | "toolConfig" + | "thinking" + | "system" + | "messages" + | "metadata" + | "output_config" + | "outputConfig" + | "requestMetadata" + | "_parallel_tool_use_config" + | "top_k" + | "topK" + | "model" + | "functions" + | "function_call" + | "parallel_tool_calls" + | "response_format" + | "n" + | "logprobs" + | "top_logprobs" + | "modalities" + | "audio" + | "prediction" + | "reasoning_effort" + | "max_completion_tokens" + | "stop" + | "max_tokens" + | "top_p" + | "frequency_penalty" + | "presence_penalty" + | "logit_bias" + | "seed" + | "stream_options" + ) }) .then_some(Unsupported("unrecognized request parameter")) } diff --git a/litellm-rust/crates/core/src/lib.rs b/litellm-rust/crates/core/src/lib.rs index 0b3573deab2..15d5c246e71 100644 --- a/litellm-rust/crates/core/src/lib.rs +++ b/litellm-rust/crates/core/src/lib.rs @@ -11,6 +11,7 @@ pub mod messages; #[cfg(any(feature = "observability", test))] pub mod observability; pub mod ocr; +pub mod params; pub mod providers; pub mod realtime; pub mod responses; diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index df9f7051011..5ae9b24d0da 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -60,6 +60,18 @@ fn provider_config_resolves_anthropic_and_azure_ai() { assert!(messages_provider_config("openai").is_none()); } +#[test] +fn native_message_extensions_survive_at_each_open_schema_boundary() { + let body = json!({"model":"claude", "messages":[{"role":"user", + "content":[{"type":"text", "text":"hi", "future_block":null, + "cache_control":{"type":"ephemeral", "future_cache":false}}], "future_message":0}], + "future_request":{"extra_body":{"timeout":null}}}); + let config = messages_provider_config("anthropic").unwrap(); + let request = serde_json::from_value(body.clone()).unwrap(); + let transformed = config.transform_request(request).unwrap(); + assert_eq!(serde_json::to_value(transformed).unwrap(), body); +} + #[test] fn truncate_error_body_caps_long_payloads() { let body = "x".repeat(400); diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/core/src/messages/types.rs index b9f807c29fd..3fd85262658 100644 --- a/litellm-rust/crates/core/src/messages/types.rs +++ b/litellm-rust/crates/core/src/messages/types.rs @@ -1,5 +1,6 @@ use std::time::Duration; +use crate::params::OpaqueFields; use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; @@ -44,7 +45,7 @@ pub struct ContentBlock { #[serde(skip_serializing_if = "Option::is_none")] pub cache_control: Option, #[serde(flatten)] - pub extra: Map, + pub extra: OpaqueFields, } #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] @@ -56,7 +57,7 @@ pub struct CacheControl { #[serde(skip_serializing_if = "Option::is_none")] pub scope: Option, #[serde(flatten)] - pub extra: Map, + pub extra: OpaqueFields, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -64,7 +65,7 @@ pub struct AnthropicMessage { pub role: String, pub content: MessageContent, #[serde(flatten)] - pub extra: Map, + pub extra: OpaqueFields, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -110,7 +111,7 @@ pub struct AnthropicMessagesRequest { #[serde(skip_serializing_if = "Option::is_none")] pub inference_geo: Option, #[serde(flatten)] - pub extra: Map, + pub extra: OpaqueFields, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] @@ -130,5 +131,5 @@ pub struct AnthropicMessagesResponse { #[serde(skip_serializing_if = "Option::is_none")] pub container: Option, #[serde(flatten)] - pub extra: Map, + pub extra: OpaqueFields, } diff --git a/litellm-rust/crates/core/src/ocr/prepare.rs b/litellm-rust/crates/core/src/ocr/prepare.rs index 9934a1d9a14..259ecdc9782 100644 --- a/litellm-rust/crates/core/src/ocr/prepare.rs +++ b/litellm-rust/crates/core/src/ocr/prepare.rs @@ -1,23 +1,18 @@ -use serde::{Deserialize, Serialize, de::DeserializeOwned}; -use serde_json::{Map, Value}; +use serde::{Serialize, de::DeserializeOwned}; +use serde_json::Value; use super::OcrClient; use super::error::{OcrError, OcrRequestError}; use super::hooks::OcrDuringCallRequest; use super::types::{LiteLLMOcrRequest, OcrDocument}; -#[derive(Debug, Deserialize)] -pub(crate) struct ParsedProviderParams { - #[serde(flatten)] - pub known: T, - #[serde(default, flatten)] - pub extra_params: Map, -} +pub(crate) use crate::params::ParsedProviderParams; #[tracing::instrument(target = "litellm::function_trace", level = "trace", skip_all)] pub(crate) fn _prepare_ocr_request( request: &LiteLLMOcrRequest, ) -> Result, OcrRequestError> { + crate::params::body_overrides(&request.optional_params).map_err(parameter_error)?; super::wire::decode_request_value( Value::Object(request.optional_params.clone()), "optional_params", @@ -26,35 +21,19 @@ pub(crate) fn _prepare_ocr_request( pub(crate) fn merge_extra_params( body: &B, - extra_params: Map, + extra_params: crate::params::OpaqueFields, ) -> Result { - let Value::Object(fields) = - serde_json::to_value(body).map_err(|_| OcrRequestError::RequestField { - path: "body".into(), - })? - else { - return Err(OcrRequestError::RequestField { - path: "body".into(), - }); - }; - let extra_body = extra_params - .get("extra_body") - .and_then(Value::as_object) - .cloned() - .unwrap_or_default() - .into_iter() - .collect::>(); - Ok(Value::Object( - fields - .into_iter() - .chain( - extra_params - .into_iter() - .filter(|(name, _)| name != "extra_body"), - ) - .chain(extra_body) - .collect(), - )) + crate::params::compose_body(body, &extra_params, &[]).map_err(parameter_error) +} + +fn parameter_error(error: crate::params::Error) -> OcrRequestError { + OcrRequestError::RequestField { + path: match error { + crate::params::Error::ExtraBody => "extra_body", + crate::params::Error::Body => "body", + } + .into(), + } } pub(crate) async fn transform_request_body( @@ -69,16 +48,26 @@ pub(crate) async fn transform_request_body( where B: Serialize + DeserializeOwned, { + let consumed = super::wire::consumed_optional_param_names( + &request.model, + Some(request.adapter.provider().as_str()), + )?; + let body = crate::params::compose_body(&body, &request.optional_params, &consumed) + .map_err(parameter_error)?; let (body, headers) = if request.hooks.intercepts_requests() { - let body = serde_json::to_value(body).map_err(|_| OcrRequestError::RequestField { - path: "body".into(), - })?; + let overrides = + crate::params::body_overrides(&request.optional_params).map_err(parameter_error)?; let retained_fields = request .optional_params .keys() .filter(|name| body.get(*name).is_some()) + .filter(|name| !overrides.is_some_and(|fields| fields.contains_key(*name))) .cloned() - .chain(retains_document.then(|| "document".to_string())) + .chain( + (retains_document + && !overrides.is_some_and(|fields| fields.contains_key("document"))) + .then(|| "document".to_string()), + ) .collect(); let changed = request .hooks @@ -91,17 +80,14 @@ where retained_fields, }) .await?; - let body = OcrWireBody::::decode(changed.body)?; - validate(&body.body)?; - (body, changed.headers) + let projected: B = + super::wire::decode_request_value(changed.body.clone(), "guardrail.body")?; + validate(&projected)?; + (changed.body, changed.headers) } else { - ( - OcrWireBody { - body, - extra: Map::new(), - }, - headers.to_vec(), - ) + let projected: B = super::wire::decode_request_value(body.clone(), "body")?; + validate(&projected)?; + (body, headers.to_vec()) }; build_http_request(client, request, url, &headers, &body) } @@ -151,38 +137,12 @@ pub(crate) async fn guardrail_document( Ok((document, changed.headers)) } -#[derive(Serialize)] -struct OcrWireBody { - #[serde(flatten)] - body: B, - #[serde(flatten)] - extra: Map, -} - -impl OcrWireBody { - fn decode(value: Value) -> Result { - let body: B = super::wire::decode_request_value(value.clone(), "guardrail.body")?; - let Value::Object(fields) = value else { - return Err(OcrRequestError::RequestField { - path: "guardrail.body".into(), - }); - }; - let known = serde_json::to_value(&body).map_err(|_| OcrRequestError::RequestField { - path: "guardrail.body".into(), - })?; - let extra = fields - .into_iter() - .filter(|(key, _)| known.get(key).is_none()) - .collect(); - Ok(Self { body, extra }) - } -} - pub(crate) fn credential_env(name: &str) -> Option { std::env::var(name).ok() } #[cfg(test)] mod tests { + use serde::Deserialize; use serde_json::json; use super::*; diff --git a/litellm-rust/crates/core/src/params.rs b/litellm-rust/crates/core/src/params.rs new file mode 100644 index 00000000000..fa82e7e2dc4 --- /dev/null +++ b/litellm-rust/crates/core/src/params.rs @@ -0,0 +1,270 @@ +use std::ops::{Deref, DerefMut}; + +use serde::{Deserialize, Serialize}; +use serde_json::{Map, Value}; + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(transparent)] +pub struct OpaqueFields(Map); + +impl Deref for OpaqueFields { + type Target = Map; + + fn deref(&self) -> &Self::Target { + &self.0 + } +} + +impl DerefMut for OpaqueFields { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.0 + } +} + +impl From> for OpaqueFields { + fn from(fields: Map) -> Self { + Self(fields) + } +} + +impl From for Map { + fn from(fields: OpaqueFields) -> Self { + fields.0 + } +} + +impl FromIterator<(String, Value)> for OpaqueFields { + fn from_iter>(fields: T) -> Self { + Self(fields.into_iter().collect()) + } +} + +#[derive(Debug, Deserialize)] +pub(crate) struct ParsedProviderParams { + #[serde(flatten)] + pub known: T, + #[serde(default, flatten)] + pub extra_params: OpaqueFields, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum Error { + #[error("extra_body must be an object")] + ExtraBody, + #[error("body must be a JSON object")] + Body, +} + +impl From for crate::Error { + fn from(error: Error) -> Self { + Self::InvalidRequest(error.to_string()) + } +} + +pub fn is_control_param(name: &str) -> bool { + matches!( + name, + "api_key" + | "api_base" + | "custom_llm_provider" + | "extra_headers" + | "extra_query" + | "timeout" + | "timeout_seconds" + | "request_timeout" + | "max_retries" + | "req_format" + | "max_response_bytes" + | "input_sources" + | "litellm_call_id" + | "litellm_logging_obj" + | "litellm_metadata" + | "proxy_server_request" + | "callbacks" + | "success_callback" + | "failure_callback" + | "guardrails" + | "drop_params" + | "additional_drop_params" + | "allowed_openai_params" + | "azure_ad_token" + | "azure_ad_token_provider" + | "tenant_id" + | "client_id" + | "client_secret" + | "azure_scope" + | "azure_authority_host" + | "azure_credential" + | "azure_federated_token_file" + | "enable_azure_ad_token_refresh" + | "vertex_credentials" + | "vertex_ai_credentials" + | "vertex_project" + | "vertex_ai_project" + | "vertex_location" + | "vertex_ai_location" + | "aws_access_key_id" + | "aws_secret_access_key" + | "aws_session_token" + | "aws_region_name" + | "aws_session_name" + | "aws_profile_name" + | "aws_role_name" + | "aws_web_identity_token" + | "aws_sts_endpoint" + | "aws_external_id" + | "aws_bedrock_runtime_endpoint" + ) +} + +pub fn body_overrides(params: &Map) -> Result>, Error> { + match params.get("extra_body") { + None | Some(Value::Null) => Ok(None), + Some(Value::Object(fields)) => Ok(Some(fields)), + Some(_) => Err(Error::ExtraBody), + } +} + +pub fn provider_fields(params: &Map) -> OpaqueFields { + params + .iter() + .filter(|(name, _)| !is_control_param(name)) + .map(|(name, value)| (name.clone(), value.clone())) + .collect() +} + +pub fn compose_body( + body: &B, + params: &Map, + consumed: &[&str], +) -> Result { + let overrides = body_overrides(params)?; + let Value::Object(fields) = serde_json::to_value(body).map_err(|_| Error::Body)? else { + return Err(Error::Body); + }; + Ok(Value::Object( + fields + .into_iter() + .chain( + params + .iter() + .filter(|(name, _)| !consumed.contains(&name.as_str())) + .chain(overrides.into_iter().flat_map(|fields| fields.iter())) + .filter(|(name, _)| name.as_str() != "extra_body" && !is_control_param(name)) + .map(|(name, value)| (name.clone(), value.clone())), + ) + .collect(), + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + use serde_json::json; + + #[rstest] + #[case::null(json!(null))] + #[case::false_value(json!(false))] + #[case::zero(json!(0))] + #[case::empty_string(json!(""))] + #[case::empty_array(json!([]))] + #[case::empty_object(json!({}))] + #[case::nested_names(json!({"timeout":null,"extra_body":{"api_key":"data"}}))] + fn unknown_values_survive_composition(#[case] value: Value) { + let params = json!({"future":value}); + assert_eq!( + compose_body(&json!({}), params.as_object().unwrap(), &[]).unwrap(), + params + ); + } + + #[test] + fn opaque_fields_do_not_apply_request_policy() { + let value = + json!({"extra_body":{"api_key":"data"},"timeout":null,"future":[false,0,"",[],{}]}); + let fields: OpaqueFields = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(fields).unwrap(), value); + } + + #[test] + fn overrides_replace_objects_without_dropping_nulls_or_nested_names() { + let params = json!({ + "future":{"extra_body":{"timeout":null}}, + "api_key":"secret", + "extra_body":{"settings":{"b":2},"explicit_null":null,"aws_secret_access_key":"secret"} + }); + assert_eq!( + compose_body( + &json!({"settings":{"a":1}}), + params.as_object().unwrap(), + &[] + ) + .unwrap(), + json!({"settings":{"b":2},"explicit_null":null,"future":{"extra_body":{"timeout":null}}}) + ); + } + + #[test] + fn consumed_options_are_not_remapped_but_overrides_are_applied() { + let params = json!({"temperature":0.2,"future":true,"extra_body":{"inferenceConfig":{"temperature":0.7}}}); + assert_eq!( + compose_body( + &json!({"inferenceConfig":{"temperature":0.2,"maxTokens":10}}), + params.as_object().unwrap(), + &["temperature"] + ) + .unwrap(), + json!({"inferenceConfig":{"temperature":0.7},"future":true}) + ); + } + + #[rstest] + #[case::absent(json!({}))] + #[case::null(json!({"extra_body":null}))] + #[case::empty(json!({"extra_body":{}}))] + fn empty_overrides(#[case] params: Value) { + assert_eq!( + compose_body(&json!({}), params.as_object().unwrap(), &[]).unwrap(), + json!({}) + ); + } + + #[rstest] + #[case::boolean(json!(false))] + #[case::number(json!(1))] + #[case::array(json!([]))] + #[case::string(json!("body"))] + fn invalid_overrides(#[case] value: Value) { + let params = json!({"extra_body":value}); + assert_eq!( + compose_body(&json!({}), params.as_object().unwrap(), &[]), + Err(Error::ExtraBody) + ); + } + + #[rstest] + fn controls_are_not_body_extensions( + #[values( + "api_key", + "timeout", + "callbacks", + "aws_secret_access_key", + "vertex_credentials", + "azure_ad_token" + )] + name: &str, + #[values(false, true)] explicit_override: bool, + ) { + let fields = Map::from_iter([(name.into(), json!("secret-or-control"))]); + let params = if explicit_override { + json!({"extra_body":fields}) + } else { + Value::Object(fields) + }; + assert_eq!( + compose_body(&json!({}), params.as_object().unwrap(), &[]).unwrap(), + json!({}) + ); + } +} diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs index b22de6c47de..ae4ec2e64d9 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/tests.rs @@ -160,7 +160,7 @@ fn accepts_an_explicit_stream_false() { } #[test] -fn declines_any_param_outside_the_allowlist() { +fn declines_known_features_requiring_python_translation() { for param in [ json!({"tools": []}), json!({"tool_choice": {"type": "auto"}}), diff --git a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs index a7d5a8ad0cf..e0043565cd8 100644 --- a/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/anthropic/chat_completions/transformation.rs @@ -146,7 +146,11 @@ impl ChatCompletionsProviderConfig for AnthropicChatCompletionsConfig { optional_params: Map, ) -> Result { Ok(ProviderChatRequestData { - body: anthropic_body(model, &build_conversation(&messages), optional_params), + body: crate::params::compose_body( + &anthropic_body(model, &build_conversation(&messages), Map::new()), + &optional_params, + &["stream"], + )?, }) } diff --git a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs index 585b34f393f..d4dd413e637 100644 --- a/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/core/src/providers/azure_ai/messages/transformation.rs @@ -86,7 +86,7 @@ fn text_content_block(text: String) -> ContentBlock { ]); ContentBlock { cache_control: None, - extra, + extra: extra.into(), } } diff --git a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs index 9bf1f73a74d..1a372028506 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/audio_transcription.rs @@ -71,17 +71,27 @@ impl AudioTranscriptionProviderConfig for BedrockAudioTranscriptionConfig { inference_config.insert("temperature".to_string(), temperature.clone()); } Ok(AudioTranscriptionRequestData { - body: json!({ - "messages": [{ - "role": "user", - "content": [ - {"audio": {"format": format, "source": {"bytes": data}}}, - {"text": instruction} - ] - }], - "system": [{"text": "You are a transcription assistant."}], - "inferenceConfig": inference_config, - }), + body: crate::params::compose_body( + &json!({ + "messages": [{ + "role": "user", + "content": [ + {"audio": {"format": format, "source": {"bytes": data}}}, + {"text": instruction} + ] + }], + "system": [{"text": "You are a transcription assistant."}], + "inferenceConfig": inference_config, + }), + &optional_params, + &[ + "language", + "prompt", + "temperature", + "response_format", + "timestamp_granularities", + ], + )?, }) } @@ -197,6 +207,30 @@ mod tests { assert_eq!(result.into_json(), json!({"text": "hello world"})); } + #[test] + fn extensions_survive_mapping_and_override_the_final_body() { + let params = json!({"language":"en", "temperature":0.1, "future":null, + "aws_secret_access_key":"secret", "extra_body":{"inferenceConfig":{"temperature":0.7}}}); + let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG + .transform_transcription_request( + "model", + json!({"data":"AQI=", "format":"wav"}), + BEDROCK_AUDIO_TRANSCRIPTION_CONFIG + .map_transcription_params(params.as_object().unwrap()), + ) + .unwrap(); + assert_eq!(result.body["inferenceConfig"], json!({"temperature":0.7})); + assert_eq!(result.body.get("future"), Some(&Value::Null)); + for name in [ + "language", + "temperature", + "extra_body", + "aws_secret_access_key", + ] { + assert!(result.body.get(name).is_none()); + } + } + #[test] fn invalid_audio_is_rejected() { let result = BEDROCK_AUDIO_TRANSCRIPTION_CONFIG.transform_transcription_request( diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs index c86f061b9ca..5d997c726c1 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/tests.rs @@ -123,7 +123,7 @@ fn declines_top_k_because_python_routes_it_by_base_model() { } #[test] -fn declines_tools_and_other_params_outside_the_allowlist() { +fn declines_known_features_requiring_python_translation() { for param in [ json!({"tools": []}), json!({"tool_choice": {"auto": {}}}), diff --git a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs index 7be3d108d44..48f24338b61 100644 --- a/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs +++ b/litellm-rust/crates/core/src/providers/bedrock/chat_completions/transformation.rs @@ -216,7 +216,15 @@ impl ChatCompletionsProviderConfig for BedrockChatCompletionsConfig { optional_params: Map, ) -> Result { Ok(ProviderChatRequestData { - body: converse_body(&build_conversation(&messages), &optional_params), + body: crate::params::compose_body( + &converse_body(&build_conversation(&messages), &optional_params), + &optional_params, + &SUPPORTED_PARAMS + .iter() + .map(|(_, name)| *name) + .chain(["stream"]) + .collect::>(), + )?, }) } diff --git a/litellm-rust/crates/core/src/realtime/types.rs b/litellm-rust/crates/core/src/realtime/types.rs index 3b59224b6e9..c4e0cca7027 100644 --- a/litellm-rust/crates/core/src/realtime/types.rs +++ b/litellm-rust/crates/core/src/realtime/types.rs @@ -1,5 +1,5 @@ +use crate::params::OpaqueFields; use serde::{Deserialize, Serialize}; -use serde_json::{Map, Value}; /// A single realtime event exchanged over the WebSocket. /// @@ -13,7 +13,7 @@ pub struct RealtimeEvent { #[serde(rename = "type")] pub event_type: String, #[serde(flatten)] - pub data: Map, + pub data: OpaqueFields, } /// One or more typed events produced by a realtime transform. @@ -34,6 +34,7 @@ impl RealtimeTransformResult { #[cfg(test)] mod tests { use super::*; + use serde_json::Value; fn event(raw: &str) -> RealtimeEvent { serde_json::from_str(raw).expect("valid event json") @@ -57,4 +58,16 @@ mod tests { let result = RealtimeTransformResult::passthrough(parsed.clone()); assert_eq!(result.events, vec![parsed]); } + + #[test] + fn native_events_do_not_interpret_sdk_extension_names() { + let raw = serde_json::json!({"type":"session.update", "extra_body":{"timeout":null}, "future":[false,0]}); + let parsed: RealtimeEvent = serde_json::from_value(raw.clone()).unwrap(); + let result = + crate::providers::openai::realtime::transformation::transform_realtime_request( + &parsed, "model", + ) + .unwrap(); + assert_eq!(serde_json::to_value(&result.events[0]).unwrap(), raw); + } } diff --git a/litellm-rust/crates/core/src/responses/types.rs b/litellm-rust/crates/core/src/responses/types.rs index 4942309992e..e8f995933f9 100644 --- a/litellm-rust/crates/core/src/responses/types.rs +++ b/litellm-rust/crates/core/src/responses/types.rs @@ -1,5 +1,6 @@ +use crate::params::OpaqueFields; use serde::{Deserialize, Deserializer, Serialize, Serializer}; -use serde_json::{Map, Value}; +use serde_json::Value; #[derive(Clone, Debug, PartialEq, Eq)] pub enum ResponsesWsEventType { @@ -58,7 +59,7 @@ pub struct ResponsesWsEvent { #[serde(rename = "type")] pub event_type: ResponsesWsEventType, #[serde(flatten)] - pub data: Map, + pub data: OpaqueFields, } impl ResponsesWsEvent { diff --git a/litellm-rust/crates/core/tests/ocr.rs b/litellm-rust/crates/core/tests/ocr.rs index 55f8713d76e..187686cdc6a 100644 --- a/litellm-rust/crates/core/tests/ocr.rs +++ b/litellm-rust/crates/core/tests/ocr.rs @@ -58,7 +58,8 @@ async fn facade_executes_direct_mistral_once() { let result = perform_ocr(wire_request( "mistral/model", &base, - json!({"pages":"0,2-4","extract_header":true,"unknown":"ignored"}), + json!({"pages":"0,2-4","extract_header":true,"unknown":{"nested":[null,false,0]}, + "extra_body":{"future":true,"extract_header":false},"timeout":42}), )) .await .unwrap(); @@ -80,7 +81,9 @@ async fn facade_executes_direct_mistral_once() { "model":"model", "document":{"type":"document_url","document_url":"data:application/pdf;base64,YWJj"}, "pages":"0,2-4", - "extract_header":true + "extract_header":false, + "unknown":{"nested":[null,false,0]}, + "future":true }) ); } diff --git a/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs index 676799eb2fe..5f6beaf7988 100644 --- a/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs +++ b/litellm-rust/crates/core/tests/vertex_ai_deepseek_ocr.rs @@ -45,7 +45,8 @@ async fn facade_executes_vertex_deepseek_at_the_openai_endpoint() { let body = request_body(&requests[0]); assert_eq!(body["model"], "deepseek-ai/deepseek-ocr-maas"); assert_eq!(body["temperature"], 0.1); - assert!(body.get("future_ocr_option").is_none()); + assert_eq!(body["future_ocr_option"], true); + assert_eq!(body["provider_option"], "value"); assert!(body.get("extra_body").is_none()); assert_eq!( body["messages"][0]["content"][0], diff --git a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs index 96a19dd62b4..fb5199d2c69 100644 --- a/litellm-rust/crates/core/tests/vertex_ai_ocr.rs +++ b/litellm-rust/crates/core/tests/vertex_ai_ocr.rs @@ -141,7 +141,8 @@ async fn adapters_build_complete_requests_and_share_mistral_normalization() { "model": "mistral-ocr-maas", "document": {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"}, "pages": [0, 2], - "include_image_base64": true + "include_image_base64": true, + "unknown": "ignored" }) ); } diff --git a/litellm-rust/crates/python-bridge/src/marshal.rs b/litellm-rust/crates/python-bridge/src/marshal.rs index 5f7633a64a0..fdfecdafe61 100644 --- a/litellm-rust/crates/python-bridge/src/marshal.rs +++ b/litellm-rust/crates/python-bridge/src/marshal.rs @@ -92,10 +92,16 @@ pub(crate) fn project_optional_fields( kwargs: &Bound<'_, PyDict>, names: &[&str], ) -> PyResult> { - names + let selected: Vec = kwargs + .py() + .import("litellm.rust_bridge.params")? + .getattr("provider_param_names")? + .call1((kwargs, names.to_vec()))? + .extract()?; + selected .iter() .filter_map(|name| match kwargs.get_item(name) { - Ok(Some(value)) => Some(from_py(&value).map(|value| ((*name).to_string(), value))), + Ok(Some(value)) => Some(from_py(&value).map(|value| (name.clone(), value))), Ok(None) => None, Err(error) => Some(Err(error)), }) diff --git a/litellm/rust_bridge/params.py b/litellm/rust_bridge/params.py new file mode 100644 index 00000000000..83626d01b3a --- /dev/null +++ b/litellm/rust_bridge/params.py @@ -0,0 +1,19 @@ +"""Select SDK extension fields without copying or converting their values.""" + +from collections.abc import Mapping, Sequence +from typing import Final + +from litellm.types.utils import all_litellm_params + + +def provider_param_names(kwargs: Mapping[str, object], consumed: Sequence[str]) -> tuple[str, ...]: + sdk_fields: Final = frozenset(all_litellm_params) | { + "model", + "document", + "timeout", + "extra_headers", + "custom_llm_provider", + "input_sources", + } + consumed_fields: Final = frozenset(consumed) + return tuple(name for name in kwargs if name in consumed_fields or name not in sdk_fields) diff --git a/tests/test_litellm/rust_bridge/test_params.py b/tests/test_litellm/rust_bridge/test_params.py new file mode 100644 index 00000000000..6cb89841a9d --- /dev/null +++ b/tests/test_litellm/rust_bridge/test_params.py @@ -0,0 +1,24 @@ +from typing import Final + +from litellm.rust_bridge.params import provider_param_names + + +def test_selects_unknown_fields_and_consumed_controls_without_reading_values() -> None: + callback: Final = object() + future: Final = {"nested": [None, False, 0]} + kwargs: Final = { + "model": "mistral/model", + "litellm_logging_obj": callback, + "metadata": callback, + "vertex_credentials": "credentials", + "future": future, + "extra_body": {"future": None}, + } + + assert provider_param_names(kwargs, ("vertex_credentials",)) == ("vertex_credentials", "future", "extra_body") + assert kwargs["future"] is future + assert kwargs["litellm_logging_obj"] is callback + + +def test_route_can_explicitly_consume_a_name_also_used_by_sdk() -> None: + assert provider_param_names({"metadata": {"provider": True}}, ("metadata",)) == ("metadata",) diff --git a/tests/test_litellm_rust/ocr/test_cohere.py b/tests/test_litellm_rust/ocr/test_cohere.py index 2a35dc62bd1..f1628deaa0d 100644 --- a/tests/test_litellm_rust/ocr/test_cohere.py +++ b/tests/test_litellm_rust/ocr/test_cohere.py @@ -40,7 +40,12 @@ async def test_public_cohere_request_and_normalization( request: Final = recording_server.requests[0] assert request.path == ("/providers/cohere/v2/parse" if model.startswith("azure_ai/") else "/v2/parse") assert request.headers["authorization"] == "Bearer test-key" - assert request.body == {"model": model.split("/", 1)[1], "document": IMAGE, "output_format": "markdown"} + assert request.body == { + "model": model.split("/", 1)[1], + "document": IMAGE, + "output_format": "markdown", + "unrecognized": True, + } assert [page.index for page in response.pages] == [4, 1] assert response.pages[0].markdown == "receipt" assert response.pages[0].images[0].bbox == BOX diff --git a/tests/test_litellm_rust/ocr/test_lifecycle.py b/tests/test_litellm_rust/ocr/test_lifecycle.py index dfcd63d3019..9173394b697 100644 --- a/tests/test_litellm_rust/ocr/test_lifecycle.py +++ b/tests/test_litellm_rust/ocr/test_lifecycle.py @@ -529,7 +529,7 @@ async def test_retained_argument_aliases_and_body_roots_survive_envelope_replace ) -> None: pages: Final = [0] document: Final = {"type": "document_url", "document_url": "data:application/pdf;base64,YWJj"} - opaque: Final = object() + opaque: Final = {"future": [None, False, 0]} observed: Final = [] class Observe(Logging): diff --git a/tests/test_litellm_rust/ocr/test_requests.py b/tests/test_litellm_rust/ocr/test_requests.py index 4f4b39fa6c6..04f142f56b5 100644 --- a/tests/test_litellm_rust/ocr/test_requests.py +++ b/tests/test_litellm_rust/ocr/test_requests.py @@ -17,6 +17,116 @@ from tests.test_litellm_rust.support.requests import ( pytestmark = pytest.mark.requires_rust_extension +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +@pytest.mark.parametrize("wrapped", [False, True], ids=["kwargs", "extra-body"]) +@pytest.mark.parametrize( + "value", + [None, False, 0, "", [], {}, {"timeout": None, "extra_body": {"api_key": "data"}}], + ids=["null", "false", "zero", "empty-string", "empty-array", "empty-object", "nested-names"], +) +async def test_native_ocr_extension_values_cross_the_boundary( + ocr_server: RecordingServer, asynchronous: bool, wrapped: bool, value: object +) -> None: + fields: Final = {"future": value} + arguments: Final = {"extra_body": fields} if wrapped else fields + if asynchronous: + await call_native_aocr(ocr_server, **arguments) + else: + call_native_ocr(ocr_server, **arguments) + assert_native_request(ocr_server) + assert ocr_server.requests[0].body == { + "model": "mistral-ocr-latest", + "document": OCR_DOCUMENT, + "future": value, + } + + +def test_native_ocr_body_hook_edits_are_not_overwritten_by_extra_body(ocr_server: RecordingServer) -> None: + observed: Final = [] + + class Edit(RecordingLogger): + def log_pre_api_call(self, model, messages, kwargs): + body = kwargs["additional_args"]["complete_input_dict"] + observed.append(body["future"]) + body["future"] = {"from": "hook"} + body.pop("removed") + + call_native_ocr( + ocr_server, + future={"from": "kwargs"}, + extra_body={"future": {"from": "extra_body"}, "removed": True}, + callbacks=[Edit()], + ) + assert observed == [{"from": "extra_body"}] + assert ocr_server.requests[0].body["future"] == {"from": "hook"} + assert "removed" not in ocr_server.requests[0].body + + +def test_native_ocr_document_override_survives_python_hook_projection(ocr_server: RecordingServer) -> None: + document: Final = {"type": "document_url", "document_url": "https://example.com/replacement.pdf"} + call_native_ocr(ocr_server, extra_body={"document": document}) + assert_native_request(ocr_server) + assert ocr_server.requests[0].body["document"] == document + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_ocr_preserves_extensions_and_applies_shallow_overrides( + ocr_server: RecordingServer, asynchronous: bool +) -> None: + future: Final = {"nested": [None, False, 0, "", [], {}], "timeout": "provider-data"} + overrides: Final = {"settings": {"b": 2}, "extract_header": False, "explicit_null": None} + arguments: Final = { + "future": future, + "settings": {"a": 1}, + "extract_header": True, + "extra_body": overrides, + } + if asynchronous: + await call_native_aocr(ocr_server, **arguments) + else: + call_native_ocr(ocr_server, **arguments) + + assert_native_request(ocr_server) + assert ocr_server.requests[0].body == { + "model": "mistral-ocr-latest", + "document": OCR_DOCUMENT, + "future": future, + "settings": {"b": 2}, + "extract_header": False, + "explicit_null": None, + } + assert arguments["future"] is future + assert arguments["extra_body"] is overrides + assert overrides == {"settings": {"b": 2}, "extract_header": False, "explicit_null": None} + + +@pytest.mark.parametrize("extra_body", [False, 1, [], "invalid"]) +def test_native_ocr_rejects_non_object_overrides_without_sending( + ocr_server: RecordingServer, extra_body: object +) -> None: + ocr_server.expected_requests = 0 + with pytest.raises(litellm.BadRequestError, match="extra_body"): + call_native_ocr(ocr_server, extra_body=extra_body) + assert ocr_server.requests == [] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_native_ocr_rejects_non_json_provider_options_without_sending( + ocr_server: RecordingServer, asynchronous: bool +) -> None: + ocr_server.expected_requests = 0 + if asynchronous: + with pytest.raises(litellm.APIConnectionError, match="unsupported type object"): + await call_native_aocr(ocr_server, future=object()) + else: + with pytest.raises(litellm.APIConnectionError, match="unsupported type object"): + call_native_ocr(ocr_server, future=object()) + assert ocr_server.requests == [] + + @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) async def test_native_azure_ocr_uses_token_provider_result_as_bearer_token( @@ -479,7 +589,7 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen from litellm.models.credentials import CredentialItem pages: Final = [0] - opaque: Final = object() + opaque: Final = {"future": [None, False, 0]} monkeypatch.setenv("MISTRAL_API_KEY", "environment-key") monkeypatch.setattr( litellm, @@ -517,6 +627,8 @@ async def test_native_ocr_inherits_named_credentials_without_overwriting_argumen assert ocr_server.requests[0].headers["authorization"] == f"Bearer {expected_key}" assert ocr_server.requests[0].body["pages"] == [0, 2] + assert ocr_server.requests[0].body["opaque"] == opaque + @pytest.mark.parametrize("source", ["sdk", "proxy"]) @pytest.mark.parametrize( diff --git a/tests/test_litellm_rust/test_ocr.py b/tests/test_litellm_rust/test_ocr.py index e0e06d685b8..6b67a277337 100644 --- a/tests/test_litellm_rust/test_ocr.py +++ b/tests/test_litellm_rust/test_ocr.py @@ -122,14 +122,14 @@ def test_native_lifecycle_core_encodes_python_file_input( document={"type": "file", "file": file_input, "mime_type": mime_type}, api_key="test-key", api_base=f"http://127.0.0.1:{server.server_port}", - opaque_extension=object(), + opaque_extension={"future": [None, False, 0]}, ) assert response.pages[0].markdown == "native OCR response" assert requests[0]["body"]["document"] == { "type": expected_type, expected_field: expected_uri, } - assert "opaque_extension" not in requests[0]["body"] + assert requests[0]["body"]["opaque_extension"] == {"future": [None, False, 0]} @pytest.mark.parametrize("asynchronous", [False, True]) diff --git a/tests/test_litellm_rust/test_request_extensions.py b/tests/test_litellm_rust/test_request_extensions.py new file mode 100644 index 00000000000..1f55741b3cc --- /dev/null +++ b/tests/test_litellm_rust/test_request_extensions.py @@ -0,0 +1,47 @@ +from typing import Final + +import pytest + +import litellm +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec + +pytestmark = pytest.mark.requires_rust_extension + + +@pytest.mark.asyncio +@pytest.mark.parametrize("asynchronous", [False, True], ids=["sync", "async"]) +async def test_completion_extensions_cross_python_rust_and_http( + recording_server: RecordingServer, asynchronous: bool +) -> None: + recording_server.default_response = ResponseSpec( + body={ + "id": "msg_test", + "type": "message", + "role": "assistant", + "model": "claude-sonnet-4-5", + "content": [{"type": "text", "text": "hello"}], + "stop_reason": "end_turn", + "stop_sequence": None, + "usage": {"input_tokens": 2, "output_tokens": 1}, + } + ) + options: Final = { + "model": "anthropic/claude-sonnet-4-5", + "messages": [{"role": "user", "content": "hi"}], + "api_base": recording_server.base_url, + "api_key": "sk-test", + "max_tokens": 16, + "future_provider_option": {"nested": [None, False, 0]}, + "extra_body": {"another_future_option": {"enabled": True}}, + } + response: Final = await litellm.acompletion(**options) if asynchronous else litellm.completion(**options) + + assert response.choices[0].message.content == "hello" + assert len(recording_server.requests) == 1 + request: Final = recording_server.requests[0] + assert not request.headers.get("user-agent", "").startswith("python-httpx") + assert request.body["future_provider_option"] == {"nested": [None, False, 0]} + assert request.body["another_future_option"] == {"enabled": True} + assert request.body["max_tokens"] == 16 + assert "extra_body" not in request.body + assert "api_key" not in request.body