diff --git a/litellm/rust/crates/inference-messages/src/handler.rs b/litellm/rust/crates/inference-messages/src/handler.rs index e1bc2322560..2ba2a3aae32 100644 --- a/litellm/rust/crates/inference-messages/src/handler.rs +++ b/litellm/rust/crates/inference-messages/src/handler.rs @@ -67,6 +67,7 @@ struct ProviderCall { struct NativeOutputFallback { input: MessagesRequest, context: MessagesTransformContext, + url: String, headers: Vec<(String, String)>, tool_custom_field: litellm_llms::base_llm::messages::tools::ToolCustomField, drops_unsupported_params: bool, @@ -83,7 +84,11 @@ impl NativeOutputFallback { &self.headers, self.drop_params, )?; - let (rebuilt, _) = config.finalize_request(rebuilt, self.headers.clone(), &self.context)?; + let (rebuilt, _) = config.finalize_request( + config.request_for_url(rebuilt, &self.url), + self.headers.clone(), + &self.context, + )?; Ok(config.wire_body(serde_json::to_value(&rebuilt).map_err(serialize_failure)?)) } } @@ -416,6 +421,7 @@ async fn provider_call( let native_output_fallback = fallback.map(|(input, context)| NativeOutputFallback { input, context, + url: url.clone(), headers: headers.clone(), tool_custom_field, drops_unsupported_params: shaping.settings.drops_unsupported_params, @@ -1982,6 +1988,68 @@ mod tests { ); } + #[tokio::test] + async fn the_native_output_retry_drops_unsigned_thinking_too() { + let model = "bedrock/invoke/us.anthropic.claude-sonnet-5-5-v1:0"; + let tool_use = json!({ + "type": "tool_use", "id": "toolu_1", "name": "Read", "input": {"file_path": "numbers.txt"} + }); + let call = provider_call_for(MessagesCall { + body: body(json!({ + "model": model, + "max_tokens": 64, + "messages": [ + {"role": "user", "content": "Read numbers.txt."}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "Read it first.", "signature": ""}, + tool_use + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "3 7 12"} + ]} + ], + "output_config": {"format": {"type": "json_schema", "schema": { + "type": "object", "properties": {"sum": {"type": "integer"}}, "required": ["sum"] + }}} + })), + api_key: None, + api_base: None, + custom_llm_provider: None, + litellm_params: crate::litellm_params(Map::from_iter([ + ("model".into(), json!(model)), + ("aws_access_key_id".into(), json!("AKIDEXAMPLE")), + ("aws_secret_access_key".into(), json!("secret")), + ("aws_region_name".into(), json!("us-east-1")), + ])) + .unwrap(), + extra_headers: None, + provider_specific_header: None, + timeout: None, + shaping: MessagesShaping { + capabilities: litellm_llms::base_llm::messages::context::MessagesModelCapabilities { + bedrock: litellm_llms::base_llm::messages::context::BedrockCapabilities { + native_output: litellm_llms::base_llm::messages::context::NativeOutputSupport { + opt_in: true, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }, + ..Default::default() + }, + }) + .await + .unwrap(); + let retried = call + .native_output_fallback + .as_ref() + .unwrap() + .body(call.provider.config()) + .unwrap(); + assert_eq!(retried["messages"][1]["content"], json!([tool_use])); + } + #[rstest] #[case::event( "data: {\"type\":\"message_stop\"}\n\n", diff --git a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs index b0cba0c9d3a..d8356fe7825 100644 --- a/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs +++ b/litellm/rust/crates/llms/src/bedrock/messages/invoke_transformations/anthropic_claude_transformation.rs @@ -2,7 +2,12 @@ use crate::anthropic::messages::transformation::retry_anthropic_messages_after_h use litellm_host::interceptors::RetriedRequest; use crate::{ - anthropic::messages::handler::shape_anthropic_messages_request, + anthropic::{ + common_utils::{ + map_messages, strip_raw_unsignable_thinking_blocks, strip_unsignable_thinking_blocks, + }, + messages::handler::shape_anthropic_messages_request, + }, base_llm::messages::context::{ MessagesShapeOptions, MessagesTransformContext, NativeOutputSupport, }, @@ -243,6 +248,17 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig { invoke_url(api_base, model, litellm_params, stream, env_lookup) } + fn request_for_url(&self, request: MessagesRequest, _url: &str) -> MessagesRequest { + MessagesRequest { + messages: map_messages( + request.messages, + strip_unsignable_thinking_blocks, + strip_raw_unsignable_thinking_blocks, + ), + ..request + } + } + /// Python's `transform_anthropic_messages_request`, which picks the betas from the caller's /// header and the request and sends them in the body. fn finalize_request( @@ -844,4 +860,48 @@ mod tests { serde_json::json!({"max_tokens": 1}) ); } + + #[test] + fn unsigned_thinking_is_dropped_and_signed_thinking_kept() { + let unsigned = + serde_json::json!({"type": "thinking", "thinking": "Read it first.", "signature": ""}); + let tool_use = serde_json::json!({ + "type": "tool_use", "id": "toolu_1", "name": "Read", "input": {"file_path": "numbers.txt"} + }); + let non_string_signature = + serde_json::json!({"type": "thinking", "thinking": "Add them.", "signature": 5}); + let unparsed_text = + serde_json::json!({"type": "text", "text": "22", "cache_control": "ephemeral"}); + let signed = serde_json::json!({"type": "thinking", "thinking": "Multiply.", "signature": "c2lnbmVk"}); + let request: MessagesRequest = serde_json::from_value(serde_json::json!({ + "model": "us.anthropic.claude-sonnet-5-5-v1:0", + "max_tokens": 64, + "messages": [ + {"role": "user", "content": "Read numbers.txt."}, + {"role": "assistant", "content": [unsigned, tool_use]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "toolu_1", "content": "3 7 12"} + ]}, + {"role": "assistant", "content": [non_string_signature, unparsed_text]}, + {"role": "user", "content": "And the product?"}, + {"role": "assistant", "content": [signed, {"type": "text", "text": "252"}]}, + {"role": "user", "content": "Thanks."} + ] + })) + .unwrap(); + let sent = serde_json::to_value(AmazonAnthropicClaudeMessagesConfig.request_for_url( + request, + "https://bedrock-runtime.us-west-2.amazonaws.com/model/x/invoke", + )) + .unwrap(); + assert_eq!( + sent["messages"][1]["content"], + serde_json::json!([tool_use]) + ); + assert_eq!( + sent["messages"][3]["content"], + serde_json::json!([unparsed_text]) + ); + assert_eq!(sent["messages"][5]["content"][0], signed); + } }