mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
fix(messages): drop unsigned thinking on bedrock invoke's rust messages route
This commit is contained in:
parent
e130ab8ca8
commit
46a843b030
2 changed files with 130 additions and 2 deletions
|
|
@ -67,6 +67,7 @@ struct ProviderCall {
|
|||
struct NativeOutputFallback {
|
||||
input: MessagesRequest,
|
||||
context: MessagesTransformContext,
|
||||
url: String,
|
||||
headers: Vec<(String, String)>,
|
||||
tool_custom_field: litellm_llms::base_llm::messages::tools::ToolCustomField,
|
||||
drops_unsupported_params: bool,
|
||||
|
|
@ -83,7 +84,11 @@ impl NativeOutputFallback {
|
|||
&self.headers,
|
||||
self.drop_params,
|
||||
)?;
|
||||
let (rebuilt, _) = config.finalize_request(rebuilt, self.headers.clone(), &self.context)?;
|
||||
let (rebuilt, _) = config.finalize_request(
|
||||
config.request_for_url(rebuilt, &self.url),
|
||||
self.headers.clone(),
|
||||
&self.context,
|
||||
)?;
|
||||
Ok(config.wire_body(serde_json::to_value(&rebuilt).map_err(serialize_failure)?))
|
||||
}
|
||||
}
|
||||
|
|
@ -416,6 +421,7 @@ async fn provider_call(
|
|||
let native_output_fallback = fallback.map(|(input, context)| NativeOutputFallback {
|
||||
input,
|
||||
context,
|
||||
url: url.clone(),
|
||||
headers: headers.clone(),
|
||||
tool_custom_field,
|
||||
drops_unsupported_params: shaping.settings.drops_unsupported_params,
|
||||
|
|
@ -1982,6 +1988,68 @@ mod tests {
|
|||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn the_native_output_retry_drops_unsigned_thinking_too() {
|
||||
let model = "bedrock/invoke/us.anthropic.claude-sonnet-5-5-v1:0";
|
||||
let tool_use = json!({
|
||||
"type": "tool_use", "id": "toolu_1", "name": "Read", "input": {"file_path": "numbers.txt"}
|
||||
});
|
||||
let call = provider_call_for(MessagesCall {
|
||||
body: body(json!({
|
||||
"model": model,
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Read numbers.txt."},
|
||||
{"role": "assistant", "content": [
|
||||
{"type": "thinking", "thinking": "Read it first.", "signature": ""},
|
||||
tool_use
|
||||
]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_1", "content": "3 7 12"}
|
||||
]}
|
||||
],
|
||||
"output_config": {"format": {"type": "json_schema", "schema": {
|
||||
"type": "object", "properties": {"sum": {"type": "integer"}}, "required": ["sum"]
|
||||
}}}
|
||||
})),
|
||||
api_key: None,
|
||||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
litellm_params: crate::litellm_params(Map::from_iter([
|
||||
("model".into(), json!(model)),
|
||||
("aws_access_key_id".into(), json!("AKIDEXAMPLE")),
|
||||
("aws_secret_access_key".into(), json!("secret")),
|
||||
("aws_region_name".into(), json!("us-east-1")),
|
||||
]))
|
||||
.unwrap(),
|
||||
extra_headers: None,
|
||||
provider_specific_header: None,
|
||||
timeout: None,
|
||||
shaping: MessagesShaping {
|
||||
capabilities: litellm_llms::base_llm::messages::context::MessagesModelCapabilities {
|
||||
bedrock: litellm_llms::base_llm::messages::context::BedrockCapabilities {
|
||||
native_output: litellm_llms::base_llm::messages::context::NativeOutputSupport {
|
||||
opt_in: true,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let retried = call
|
||||
.native_output_fallback
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.body(call.provider.config())
|
||||
.unwrap();
|
||||
assert_eq!(retried["messages"][1]["content"], json!([tool_use]));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::event(
|
||||
"data: {\"type\":\"message_stop\"}\n\n",
|
||||
|
|
|
|||
|
|
@ -2,7 +2,12 @@ use crate::anthropic::messages::transformation::retry_anthropic_messages_after_h
|
|||
use litellm_host::interceptors::RetriedRequest;
|
||||
|
||||
use crate::{
|
||||
anthropic::messages::handler::shape_anthropic_messages_request,
|
||||
anthropic::{
|
||||
common_utils::{
|
||||
map_messages, strip_raw_unsignable_thinking_blocks, strip_unsignable_thinking_blocks,
|
||||
},
|
||||
messages::handler::shape_anthropic_messages_request,
|
||||
},
|
||||
base_llm::messages::context::{
|
||||
MessagesShapeOptions, MessagesTransformContext, NativeOutputSupport,
|
||||
},
|
||||
|
|
@ -243,6 +248,17 @@ impl BaseMessagesConfig for AmazonAnthropicClaudeMessagesConfig {
|
|||
invoke_url(api_base, model, litellm_params, stream, env_lookup)
|
||||
}
|
||||
|
||||
fn request_for_url(&self, request: MessagesRequest, _url: &str) -> MessagesRequest {
|
||||
MessagesRequest {
|
||||
messages: map_messages(
|
||||
request.messages,
|
||||
strip_unsignable_thinking_blocks,
|
||||
strip_raw_unsignable_thinking_blocks,
|
||||
),
|
||||
..request
|
||||
}
|
||||
}
|
||||
|
||||
/// Python's `transform_anthropic_messages_request`, which picks the betas from the caller's
|
||||
/// header and the request and sends them in the body.
|
||||
fn finalize_request(
|
||||
|
|
@ -844,4 +860,48 @@ mod tests {
|
|||
serde_json::json!({"max_tokens": 1})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsigned_thinking_is_dropped_and_signed_thinking_kept() {
|
||||
let unsigned =
|
||||
serde_json::json!({"type": "thinking", "thinking": "Read it first.", "signature": ""});
|
||||
let tool_use = serde_json::json!({
|
||||
"type": "tool_use", "id": "toolu_1", "name": "Read", "input": {"file_path": "numbers.txt"}
|
||||
});
|
||||
let non_string_signature =
|
||||
serde_json::json!({"type": "thinking", "thinking": "Add them.", "signature": 5});
|
||||
let unparsed_text =
|
||||
serde_json::json!({"type": "text", "text": "22", "cache_control": "ephemeral"});
|
||||
let signed = serde_json::json!({"type": "thinking", "thinking": "Multiply.", "signature": "c2lnbmVk"});
|
||||
let request: MessagesRequest = serde_json::from_value(serde_json::json!({
|
||||
"model": "us.anthropic.claude-sonnet-5-5-v1:0",
|
||||
"max_tokens": 64,
|
||||
"messages": [
|
||||
{"role": "user", "content": "Read numbers.txt."},
|
||||
{"role": "assistant", "content": [unsigned, tool_use]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "toolu_1", "content": "3 7 12"}
|
||||
]},
|
||||
{"role": "assistant", "content": [non_string_signature, unparsed_text]},
|
||||
{"role": "user", "content": "And the product?"},
|
||||
{"role": "assistant", "content": [signed, {"type": "text", "text": "252"}]},
|
||||
{"role": "user", "content": "Thanks."}
|
||||
]
|
||||
}))
|
||||
.unwrap();
|
||||
let sent = serde_json::to_value(AmazonAnthropicClaudeMessagesConfig.request_for_url(
|
||||
request,
|
||||
"https://bedrock-runtime.us-west-2.amazonaws.com/model/x/invoke",
|
||||
))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
sent["messages"][1]["content"],
|
||||
serde_json::json!([tool_use])
|
||||
);
|
||||
assert_eq!(
|
||||
sent["messages"][3]["content"],
|
||||
serde_json::json!([unparsed_text])
|
||||
);
|
||||
assert_eq!(sent["messages"][5]["content"][0], signed);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue