fix(messages): drop unsigned thinking on bedrock invoke's rust messages route

This commit is contained in:
mateo-berri 2026-11-15 20:01:50 -08:00
parent e130ab8ca8
commit 46a843b030
2 changed files with 130 additions and 2 deletions

View file

@ -67,6 +67,7 @@ struct ProviderCall {
struct NativeOutputFallback {
input: MessagesRequest,
context: MessagesTransformContext,
url: String,
headers: Vec<(String, String)>,
tool_custom_field: litellm_llms::base_llm::messages::tools::ToolCustomField,
drops_unsupported_params: bool,
@ -83,7 +84,11 @@ impl NativeOutputFallback {
&self.headers,
self.drop_params,
)?;
let (rebuilt, _) = config.finalize_request(rebuilt, self.headers.clone(), &self.context)?;
let (rebuilt, _) = config.finalize_request(
config.request_for_url(rebuilt, &self.url),
self.headers.clone(),
&self.context,
)?;
Ok(config.wire_body(serde_json::to_value(&rebuilt).map_err(serialize_failure)?))
}
}
@ -416,6 +421,7 @@ async fn provider_call(
let native_output_fallback = fallback.map(|(input, context)| NativeOutputFallback {
input,
context,
url: url.clone(),
headers: headers.clone(),
tool_custom_field,
drops_unsupported_params: shaping.settings.drops_unsupported_params,
@ -1982,6 +1988,68 @@ mod tests {
);
}
#[tokio::test]
async fn the_native_output_retry_drops_unsigned_thinking_too() {
let model = "bedrock/invoke/us.anthropic.claude-sonnet-5-5-v1:0";
let tool_use = json!({
"type": "tool_use", "id": "toolu_1", "name": "Read", "input": {"file_path": "numbers.txt"}
});
let call = provider_call_for(MessagesCall {
body: body(json!({
"model": model,
"max_tokens": 64,
"messages": [
{"role": "user", "content": "Read numbers.txt."},
{"role": "assistant", "content": [
{"type": "thinking", "thinking": "Read it first.", "signature": ""},
tool_use
]},
{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "toolu_1", "content": "3 7 12"}
]}
],
"output_config": {"format": {"type": "json_schema", "schema": {
"type": "object", "properties": {"sum": {"type": "integer"}}, "required": ["sum"]
}}}
})),
api_key: None,
api_base: None,
custom_llm_provider: None,
litellm_params: crate::litellm_params(Map::from_iter([
("model".into(), json!(model)),
("aws_access_key_id".into(), json!("AKIDEXAMPLE")),
("aws_secret_access_key".into(), json!("secret")),
("aws_region_name".into(), json!("us-east-1")),
]))
.unwrap(),
extra_headers: None,
provider_specific_header: None,
timeout: None,
shaping: MessagesShaping {
capabilities: litellm_llms::base_llm::messages::context::MessagesModelCapabilities {
bedrock: litellm_llms::base_llm::messages::context::BedrockCapabilities {
native_output: litellm_llms::base_llm::messages::context::NativeOutputSupport {
opt_in: true,
..Default::default()
},
..Default::default()
},
..Default::default()
},
..Default::default()
},
})
.await
.unwrap();
let retried = call
.native_output_fallback
.as_ref()
.unwrap()
.body(call.provider.config())
.unwrap();
assert_eq!(retried["messages"][1]["content"], json!([tool_use]));
}
#[rstest]
#[case::event(
"data: {\"type\":\"message_stop\"}\n\n",

View file

@ -2,7 +2,12 @@ use crate::anthropic::messages::transformation::retry_anthropic_messages_after_h
use litellm_host::interceptors::RetriedRequest;
use crate::{
anthropic::messages::handler::shape_anthropic_messages_request,
anthropic::{
common_utils::{
map_messages, strip_raw_unsignable_thinking_blocks, strip_unsignable_thinking_blocks,
},
messages::handler::shape_anthropic_messages_request,
},
base_llm::messages::context::{
MessagesShapeOptions, MessagesTransformContext, NativeOutputSupport,
},
@ -243,6 +248,17 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
invoke_url(api_base, model, litellm_params, stream, env_lookup)
}
fn request_for_url(&self, request: MessagesRequest, _url: &str) -> MessagesRequest {
MessagesRequest {
messages: map_messages(
request.messages,
strip_unsignable_thinking_blocks,
strip_raw_unsignable_thinking_blocks,
),
..request
}
}
/// Python's `transform_anthropic_messages_request`, which picks the betas from the caller's
/// header and the request and sends them in the body.
fn finalize_request(
@ -844,4 +860,48 @@ mod tests {
serde_json::json!({"max_tokens": 1})
);
}
#[test]
fn unsigned_thinking_is_dropped_and_signed_thinking_kept() {
let unsigned =
serde_json::json!({"type": "thinking", "thinking": "Read it first.", "signature": ""});
let tool_use = serde_json::json!({
"type": "tool_use", "id": "toolu_1", "name": "Read", "input": {"file_path": "numbers.txt"}
});
let non_string_signature =
serde_json::json!({"type": "thinking", "thinking": "Add them.", "signature": 5});
let unparsed_text =
serde_json::json!({"type": "text", "text": "22", "cache_control": "ephemeral"});
let signed = serde_json::json!({"type": "thinking", "thinking": "Multiply.", "signature": "c2lnbmVk"});
let request: MessagesRequest = serde_json::from_value(serde_json::json!({
"model": "us.anthropic.claude-sonnet-5-5-v1:0",
"max_tokens": 64,
"messages": [
{"role": "user", "content": "Read numbers.txt."},
{"role": "assistant", "content": [unsigned, tool_use]},
{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "toolu_1", "content": "3 7 12"}
]},
{"role": "assistant", "content": [non_string_signature, unparsed_text]},
{"role": "user", "content": "And the product?"},
{"role": "assistant", "content": [signed, {"type": "text", "text": "252"}]},
{"role": "user", "content": "Thanks."}
]
}))
.unwrap();
let sent = serde_json::to_value(AmazonAnthropicClaudeMessagesConfig.request_for_url(
request,
"https://bedrock-runtime.us-west-2.amazonaws.com/model/x/invoke",
))
.unwrap();
assert_eq!(
sent["messages"][1]["content"],
serde_json::json!([tool_use])
);
assert_eq!(
sent["messages"][3]["content"],
serde_json::json!([unparsed_text])
);
assert_eq!(sent["messages"][5]["content"][0], signed);
}
}