diff --git a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs index df1ae13fd28..7f22acc18db 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/thinking.rs @@ -306,20 +306,53 @@ fn translate_adaptive_effort_for_non_adaptive_model( }) } +pub(crate) fn drop_unsupported_effort( + request: MessagesRequest, + capabilities: &MessagesModelCapabilities, +) -> MessagesRequest { + let rejected = |level: EffortLevel| !accepts_effort(capabilities, level); + let drops_reasoning_effort = capabilities.supports_adaptive_thinking + && matches!( + &request.params.reasoning_effort, + Some(Recognized::Known(effort)) if output_effort(*effort).is_some_and(rejected) + ); + let reshaped_to_budget = !capabilities.supports_adaptive_thinking + && matches!(known_thinking(&request), Some(ThinkingConfig::Adaptive(_))); + let drops_output_effort = supports_effort_param(capabilities) + && !reshaped_to_budget + && matches!(known_effort(&request), Some(Recognized::Known(level)) if rejected(*level)); + MessagesRequest { + params: MessagesOptionalParams { + reasoning_effort: request + .params + .reasoning_effort + .filter(|_| !drops_reasoning_effort), + output_config: if drops_output_effort { + without_effort(request.params.output_config) + } else { + request.params.output_config + }, + ..request.params + }, + ..request + } +} + fn drop_incompatible_temperature_for_thinking( request: MessagesRequest, context: &ThinkingContext, ) -> MessagesRequest { - if context.capabilities.supports_adaptive_thinking { - return request; - } let pinned = request .params .temperature .is_some_and(|temperature| temperature != 1.0); - let thinking_enabled = matches!(known_thinking(&request), Some(ThinkingConfig::Enabled(_))); - let effort_enabled = known_effort(&request).is_some(); - if !pinned || !(thinking_enabled || effort_enabled) { + let thinking_on = matches!( + known_thinking(&request), + Some(ThinkingConfig::Enabled(_) | ThinkingConfig::Adaptive(_)) + ); + let effort_on_budget_model = + !context.capabilities.supports_adaptive_thinking && known_effort(&request).is_some(); + if !pinned || !(thinking_on || effort_on_budget_model) { return request; } MessagesRequest { @@ -1092,6 +1125,47 @@ mod tests { ); } + #[rstest] + #[case::claude_code_payload( + opus_4_7(), + claude_code_payload("medium", 8192), + 0.0, + claude_code_payload("medium", 8192) + )] + #[case::legacy_thinking_kept_on_legacy_capable_model( + sonnet_4_6(), + serde_json::json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}), + 0.5, + serde_json::json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + #[case::bare_adaptive_thinking( + sonnet_4_6(), + serde_json::json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}), + 0.5, + serde_json::json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}) + )] + #[case::reasoning_effort_synthesized_adaptive_thinking( + sonnet_4_6(), + serde_json::json!({"max_tokens": 8192, "reasoning_effort": "high"}), + 0.0, + serde_json::json!({ + "max_tokens": 8192, + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": "high"} + }) + )] + fn pinned_temperature_is_dropped_when_thinking_survives_on_adaptive_model( + #[case] capabilities: MessagesModelCapabilities, + #[case] input: Value, + #[case] temperature: f64, + #[case] expected: Value, + ) { + assert_eq!( + translate(capabilities, with_temperature(input, temperature)), + Ok(request(expected)) + ); + } + #[rstest] #[case::temperature_one_with_enabled_thinking( haiku_4_5(), @@ -1124,17 +1198,17 @@ mod tests { 0.0, serde_json::json!({"max_tokens": 8192, "output_config": {"format": {"type": "json_schema"}}}) )] - #[case::adaptive_model( - opus_4_7(), - claude_code_payload("medium", 8192), - 0.0, - claude_code_payload("medium", 8192) - )] - #[case::legacy_thinking_on_adaptive_model( + #[case::effort_without_thinking_on_adaptive_model( sonnet_4_6(), - serde_json::json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}), - 0.0, - serde_json::json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + serde_json::json!({"max_tokens": 8192, "output_config": {"effort": "high"}}), + 0.5, + serde_json::json!({"max_tokens": 8192, "output_config": {"effort": "high"}}) + )] + #[case::disabled_thinking_on_adaptive_model( + sonnet_4_6(), + serde_json::json!({"max_tokens": 8192, "thinking": {"type": "disabled"}}), + 0.5, + serde_json::json!({"max_tokens": 8192, "thinking": {"type": "disabled"}}) )] fn temperature_is_kept( #[case] capabilities: MessagesModelCapabilities, diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 6657cbf262f..c4b57a64e43 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -9,7 +9,10 @@ use litellm_llms_types::{ use litellm_router_types::LitellmParams; use serde_json::{Map, Value, json}; -use super::{handler::shape_anthropic_messages_request, thinking::translate_thinking}; +use super::{ + handler::shape_anthropic_messages_request, + thinking::{drop_unsupported_effort, translate_thinking}, +}; use crate::base_llm::messages::context::MessagesTransformContext; use crate::{ Error, @@ -267,6 +270,11 @@ fn drop_unsupported_params( context: &MessagesTransformContext, ) -> Result { let capabilities = &context.thinking.capabilities; + let request = if context.drop_params { + drop_unsupported_effort(request, capabilities) + } else { + request + }; let model = request.model.clone(); let reject = |param: &str, value: String, hint: &str| -> Result<(), Error> { if context.drop_params { @@ -380,6 +388,7 @@ mod tests { use super::*; use crate::anthropic::common_utils::ENCRYPTED_REASONING_SIGNATURE_PREFIX; + use crate::base_llm::messages::context::SupportedEffortTiers; type Env = &'static [(&'static str, &'static str)]; @@ -481,6 +490,30 @@ mod tests { } } + #[fixture] + fn adaptive_without_xhigh() -> MessagesModelCapabilities { + MessagesModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_legacy_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + max: true, + ..Default::default() + }, + ..Default::default() + } + } + + #[fixture] + fn effort_without_adaptive() -> MessagesModelCapabilities { + MessagesModelCapabilities { + supports_reasoning: true, + supports_output_config: true, + ..Default::default() + } + } + #[rstest] #[case::alone(json!({"max_tokens": null}))] #[case::ahead_of_the_param_gate(json!({"max_tokens": null, "speed": "fast"}))] @@ -514,6 +547,16 @@ mod tests { )] #[case::speed_on_a_fast_mode_model(fast_mode(), false, json!({"speed": "fast"}))] #[case::speed_on_a_fast_mode_model_under_drop_params(fast_mode(), true, json!({"speed": "fast"}))] + #[case::effort_tier_the_model_lacks_is_left_to_the_provider( + adaptive_without_xhigh(), + false, + json!({"output_config": {"effort": "xhigh"}}) + )] + #[case::effort_tier_the_model_has_under_drop_params( + adaptive_without_xhigh(), + true, + json!({"output_config": {"effort": "max"}}) + )] #[case::native_context_management_edits(unmapped(), false, json!({"context_management": {"edits": [{ "type": "clear_tool_uses_20250919", "trigger": {"type": "input_tokens", "value": 30000}, @@ -568,6 +611,21 @@ mod tests { json!({"speed": "fast", "temperature": 1.0}), json!({"temperature": 1.0}) )] + #[case::reasoning_effort_tier_the_adaptive_model_lacks( + adaptive_without_xhigh(), + json!({"reasoning_effort": "xhigh"}), + json!({}) + )] + #[case::output_config_effort_tier_the_adaptive_model_lacks( + adaptive_without_xhigh(), + json!({"output_config": {"effort": "xhigh", "format": {"type": "json_schema"}}}), + json!({"output_config": {"format": {"type": "json_schema"}}}) + )] + #[case::output_config_effort_tier_the_effort_model_lacks( + effort_without_adaptive(), + json!({"output_config": {"effort": "max"}}), + json!({}) + )] fn removed_params_are_dropped_under_drop_params( #[case] capabilities: MessagesModelCapabilities, #[case] fields: Value, @@ -576,6 +634,25 @@ mod tests { assert_eq!(transform(fields, capabilities, true), Ok(body(expected))); } + #[rstest] + fn drop_params_keeps_the_effort_sized_budget_on_a_model_without_adaptive_thinking( + effort_without_adaptive: MessagesModelCapabilities, + ) { + let adaptive = json!({"max_tokens": 64000, "thinking": {"type": "adaptive"}}); + let with_xhigh = merged( + adaptive.clone(), + json!({"output_config": {"effort": "xhigh"}}), + ); + + let dropped = transform(with_xhigh.clone(), effort_without_adaptive, true); + + assert_eq!( + dropped, + transform(with_xhigh, effort_without_adaptive, false) + ); + assert_ne!(dropped, transform(adaptive, effort_without_adaptive, true)); + } + #[rstest] #[case::temperature( sampling_removed(), diff --git a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs index bf934fb4d4c..ec4e75f7ed3 100644 --- a/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/messages/transformation.rs @@ -363,7 +363,7 @@ mod tests { "metadata": {"user_id": "u1"}, "stop_sequences": ["STOP"], "stream": false, - "temperature": 0.4, + "temperature": 1.0, "top_p": 0.9, "top_k": 40, "tools": [{"name": "get_weather", "input_schema": {"type": "object"}}],