diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ed658e074ee..310fa4d8056 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4292,6 +4292,7 @@ dependencies = [ "base64 0.22.1", "bytes", "data-url", + "fancy-regex 0.19.2", "futures-util", "litellm-auth", "litellm-auth-aws", diff --git a/litellm-rust/crates/inference-messages/src/handler.rs b/litellm-rust/crates/inference-messages/src/handler.rs index 609fa67484a..7c39278e17b 100644 --- a/litellm-rust/crates/inference-messages/src/handler.rs +++ b/litellm-rust/crates/inference-messages/src/handler.rs @@ -132,8 +132,10 @@ async fn provider_call( MessagesRequest { model, ..body }, shaping.settings.reasoning_auto_summary, )?; - let transform_context = - MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params); + let transform_context = MessagesTransformContext { + drops_unsupported_params: shaping.settings.drops_unsupported_params, + ..MessagesTransformContext::new(shaping.capabilities, shaping.settings.drop_params) + }; let without_listed = without_additional_drop_params(sanitized, &shaping.settings.additional_drop_params)?; let trimmed = requested_optional_params(without_listed, provider, &transform_context)?; diff --git a/litellm-rust/crates/inference-messages/src/types.rs b/litellm-rust/crates/inference-messages/src/types.rs index 772ea6f7292..e35a6a5e272 100644 --- a/litellm-rust/crates/inference-messages/src/types.rs +++ b/litellm-rust/crates/inference-messages/src/types.rs @@ -71,6 +71,8 @@ pub struct MessagesSettings { pub additional_drop_params: Vec, #[serde(default)] pub request_timeout: Option, + #[serde(default)] + pub drops_unsupported_params: bool, } impl MessagesSettings { @@ -140,7 +142,8 @@ mod tests { "drop_params": true, "reasoning_auto_summary": true, "additional_drop_params": ["metadata.user_id", "thinking"], - "request_timeout": 1800.0 + "request_timeout": 1800.0, + "drops_unsupported_params": true }), MessagesShaping { settings: MessagesSettings { @@ -148,6 +151,7 @@ mod tests { reasoning_auto_summary: true, additional_drop_params: vec!["metadata.user_id".to_string(), "thinking".to_string()], request_timeout: Some(1800.0), + drops_unsupported_params: true, }, capabilities: MessagesModelCapabilities { supports_reasoning: true, @@ -185,6 +189,10 @@ mod tests { serialized["reasoning_auto_summary"], json!(expected.settings.reasoning_auto_summary) ); + assert_eq!( + serialized["drops_unsupported_params"], + json!(expected.settings.drops_unsupported_params) + ); assert_eq!( serialized["additional_drop_params"], json!(expected.settings.additional_drop_params) diff --git a/litellm-rust/crates/inference-messages/tests/messages/retry.rs b/litellm-rust/crates/inference-messages/tests/messages/retry.rs index a7df57551a9..e48482dc8b0 100644 --- a/litellm-rust/crates/inference-messages/tests/messages/retry.rs +++ b/litellm-rust/crates/inference-messages/tests/messages/retry.rs @@ -216,3 +216,149 @@ async fn other_failures_reach_the_caller_without_a_retry( assert!(host.retried.lock().unwrap().is_empty()); assert!(observed_retries(&host).is_empty()); } + +const STRICT_REJECTION: &str = "Too many strict tools (21). The maximum is 20."; + +fn strict_tools( + call: MessagesCall, + api_base: String, + stream: bool, + drop_params: bool, +) -> MessagesCall { + MessagesCall { + api_key: Some("sk-ant".into()), + api_base: Some(api_base), + shaping: MessagesShaping { + settings: MessagesSettings { + drop_params, + drops_unsupported_params: drop_params, + ..MessagesSettings::default() + }, + ..MessagesShaping::default() + }, + ..with_fields( + call, + json!({ + "max_tokens": 64, + "stream": stream, + "messages": [{"role": "user", "content": "hi"}], + "tools": [ + {"name": "a", "input_schema": {"type": "object"}, "strict": true}, + {"name": "b", "input_schema": {"type": "object"}, "strict": false}, + {"name": "c", "input_schema": {"type": "object"}} + ] + }), + ) + } +} + +#[rstest] +#[tokio::test] +async fn a_rejected_strict_tool_is_sent_once_more_without_strict_under_drop_params( + call: MessagesCall, + #[values(false, true)] stream: bool, +) { + let success = if stream { + sse_response() + } else { + message_response() + }; + let upstream = upstream([rejection(400, STRICT_REJECTION), success]).await; + let host = RecordingCall::::new(strict_tools(call, upstream.uri(), stream, true)); + + let (response, _) = execute(&host).await.expect("the retry succeeds"); + + let [first, second] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + assert_eq!(first.json()["tools"][0]["strict"], json!(true)); + let mut expected = first.json(); + expected["tools"] = json!([ + {"name": "a", "input_schema": {"type": "object"}}, + {"name": "b", "input_schema": {"type": "object"}}, + {"name": "c", "input_schema": {"type": "object"}} + ]); + assert_eq!(second.json(), expected); + let expected_retry = RetriedRequest { + reasons: vec![RetryReason::StrictTools { dropped: 1 }], + body: second.json(), + }; + assert_eq!(*host.retried.lock().unwrap(), vec![expected_retry.clone()]); + assert_eq!(observed_retries(&host), vec![expected_retry]); + assert_eq!( + matches!(response, MessagesCallResponse::Stream { .. }), + stream + ); +} + +#[rstest] +#[tokio::test] +async fn a_rejected_strict_tool_reaches_the_caller_without_drop_params(call: MessagesCall) { + let upstream = upstream([rejection(400, STRICT_REJECTION), message_response()]).await; + let host = RecordingCall::::new(strict_tools(call, upstream.uri(), false, false)); + + let error = execute(&host).await.err().expect("the call fails"); + + assert_eq!( + only_request(&upstream).await.json()["tools"][0]["strict"], + json!(true) + ); + assert_eq!(error.upstream_response().unwrap().0, 400); + assert!(observed_retries(&host).is_empty()); +} + +#[rstest] +#[tokio::test] +async fn strict_tools_and_a_thinking_replay_share_the_one_retry(call: MessagesCall) { + let upstream = upstream([ + rejection(400, STRICT_REJECTION), + rejection(400, INVALID_SIGNATURE), + message_response(), + ]) + .await; + let host = RecordingCall::::new(replay( + strict_tools(call, upstream.uri(), false, true), + upstream.uri(), + false, + enabled(), + )); + + let error = execute(&host) + .await + .err() + .expect("the second rejection is raised"); + + let [first, second] = <[_; 2]>::try_from(received(&upstream).await).unwrap(); + assert_eq!(first.json()["thinking"], enabled()); + assert_eq!(second.json()["thinking"], enabled()); + assert_eq!( + second.json()["tools"][0], + json!({"name": "a", "input_schema": {"type": "object"}}) + ); + assert_eq!(received(&upstream).await.len(), 2); + assert!(error.upstream_response().unwrap().1.contains("signature")); +} + +#[rstest] +#[tokio::test] +async fn a_truthy_global_drop_params_that_is_not_true_does_not_drop_strict(call: MessagesCall) { + let upstream = upstream([rejection(400, STRICT_REJECTION), message_response()]).await; + let strict = strict_tools(call, upstream.uri(), false, true); + let host = RecordingCall::::new(MessagesCall { + shaping: MessagesShaping { + settings: MessagesSettings { + drops_unsupported_params: false, + ..strict.shaping.settings.clone() + }, + ..strict.shaping.clone() + }, + ..strict + }); + + let error = execute(&host).await.err().expect("the call fails"); + + assert_eq!( + only_request(&upstream).await.json()["tools"][0]["strict"], + json!(true) + ); + assert_eq!(error.upstream_response().unwrap().0, 400); + assert!(observed_retries(&host).is_empty()); +} diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 79ee17e8b97..29666f2c7e9 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -26,6 +26,7 @@ litellm-router-types.workspace = true base64.workspace = true bytes.workspace = true data-url = "0.3.2" +fancy-regex.workspace = true futures-util.workspace = true reqwest.workspace = true serde.workspace = true diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs index f11416ea720..ab043bf8f17 100644 --- a/litellm-rust/crates/llms/src/anthropic/common_utils.rs +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -1,4 +1,7 @@ +use std::sync::LazyLock; + use crate::base_llm::messages::context::MessagesModelCapabilities; +use fancy_regex::Regex; use litellm_auth::CredentialPlacement; use litellm_secret_value::SecretValue; @@ -706,6 +709,57 @@ pub fn strip_thinking_blocks_from_anthropic_messages_request(request_body: Value Value::Object(body) } +static CLAUDE_STRICT_TOOLS_REJECTION: LazyLock = LazyLock::new(|| { + Regex::new(concat!( + r"Too many strict tools \(\d+\)", + r"|Schemas contains too many (?:optional parameters|parameters with union types)", + r"|[Cc]ompiled grammar (?:is too large|size)", + r"|tools\.\d+\.custom(?:\.strict)?: ", + r"|Grammar compilation timed out", + )) + .expect("valid regex") +}); + +pub fn is_claude_strict_tools_rejection(status: u16, error_text: &str) -> bool { + status == 400 + && CLAUDE_STRICT_TOOLS_REJECTION + .is_match(error_text) + .unwrap_or(false) +} + +#[derive(Clone, Debug, PartialEq)] +pub struct ToolsWithoutStrict { + pub tools: Vec, + pub dropped_count: usize, +} + +pub fn tools_without_strict(tools: Option<&Value>) -> Option { + let tools = tools?.as_array()?; + if !tools.iter().all(Value::is_object) { + return None; + } + let dropped_count = tools + .iter() + .filter(|tool| tool.get("strict") == Some(&Value::Bool(true))) + .count(); + if dropped_count == 0 { + return None; + } + Some(ToolsWithoutStrict { + tools: tools + .iter() + .cloned() + .map(|mut tool| { + if let Value::Object(fields) = &mut tool { + fields.shift_remove("strict"); + } + tool + }) + .collect(), + dropped_count, + }) +} + #[cfg(test)] mod tests { use crate::base_llm::messages::context::SupportedEffortTiers; @@ -2141,4 +2195,55 @@ mod tests { .collect(); assert_eq!(keys, ["model", "messages", "max_tokens"]); } + + fn shared_table(table: &str) -> Vec { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + fixture[table].as_array().unwrap().clone() + } + + #[test] + fn strict_tools_rejections_match_the_cases_shared_with_python() { + let cases = shared_table("claude_strict_tools_rejection"); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + is_claude_strict_tools_rejection(status, case["error_text"].as_str().unwrap()) + != case["expected"].as_bool().unwrap() + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } + + #[test] + fn tools_without_strict_matches_the_cases_shared_with_python() { + let cases = shared_table("tools_without_strict"); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let expected = case["expected"] + .as_object() + .map(|expected| ToolsWithoutStrict { + tools: expected["tools"].as_array().unwrap().clone(), + dropped_count: usize::try_from(expected["dropped_count"].as_u64().unwrap()) + .unwrap(), + }); + tools_without_strict(Some(&case["tools"])) != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } + + #[test] + fn absent_tools_have_no_strict_to_drop() { + assert_eq!(tools_without_strict(None), None); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs index 6dc67b5b1d5..50c17cbd5d0 100644 --- a/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/messages/transformation.rs @@ -21,9 +21,10 @@ use crate::{ anthropic_request_errors_on_thinking_binding_mismatch, complete_anthropic_url, get_auth_header, has_advisor_tool, has_anthropic_credential, is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error, - is_tool_search_used, merge_beta_headers, optionally_handle_anthropic_oauth, - requires_native_compaction_beta, strip_advisor_blocks, strip_encrypted_reasoning_blocks, - strip_thinking_blocks_from_anthropic_messages_request, + is_claude_strict_tools_rejection, is_tool_search_used, merge_beta_headers, + optionally_handle_anthropic_oauth, requires_native_compaction_beta, strip_advisor_blocks, + strip_encrypted_reasoning_blocks, strip_thinking_blocks_from_anthropic_messages_request, + tools_without_strict, }, base_llm::{ auth::AuthScheme, @@ -136,11 +137,17 @@ pub fn should_retry_anthropic_messages_on_http_error( status: u16, error_text: &str, request_body: &Value, - _context: &MessagesTransformContext, + context: &MessagesTransformContext, ) -> bool { if status != 400 { return false; } + let drops_rejected_strict_tools = context.drops_unsupported_params + && is_claude_strict_tools_rejection(status, error_text) + && tools_without_strict(request_body.get("tools")).is_some(); + if drops_rejected_strict_tools { + return true; + } let caller_asked_for_this_error = anthropic_request_errors_on_thinking_binding_mismatch(request_body) && is_anthropic_thinking_binding_mismatch_error(error_text); @@ -152,16 +159,25 @@ pub fn transform_anthropic_messages_request_on_http_error( error_text: &str, request_body: Value, ) -> RetriedRequest { - if status == 400 && is_anthropic_invalid_thinking_block_error(error_text) { - return RetriedRequest { - reasons: vec![RetryReason::ThinkingSignature], - body: strip_thinking_blocks_from_anthropic_messages_request(request_body), - }; - } - RetriedRequest { - reasons: Vec::new(), - body: request_body, + let mut reasons = Vec::new(); + let mut body = if status == 400 && is_anthropic_invalid_thinking_block_error(error_text) { + reasons.push(RetryReason::ThinkingSignature); + strip_thinking_blocks_from_anthropic_messages_request(request_body) + } else { + request_body + }; + let without_strict = if is_claude_strict_tools_rejection(status, error_text) { + tools_without_strict(body.get("tools")) + } else { + None + }; + if let (Some(without_strict), Value::Object(fields)) = (without_strict, &mut body) { + reasons.push(RetryReason::StrictTools { + dropped: without_strict.dropped_count, + }); + fields.insert("tools".to_owned(), Value::Array(without_strict.tools)); } + RetriedRequest { reasons, body } } pub fn retry_anthropic_messages_after_http_error( @@ -927,6 +943,7 @@ mod tests { keeps_display_updates: false, }, drop_params: true, + drops_unsupported_params: false, } ); return; @@ -1462,4 +1479,59 @@ mod tests { .collect(); assert_eq!(mismatched, Vec::<&str>::new()); } + + #[test] + fn the_strict_tools_retry_matches_the_cases_shared_with_python() { + let fixture: Value = serde_json::from_str(include_str!(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/fixtures/anthropic_messages_http_error_retry.json" + ))) + .unwrap(); + let replay = &fixture["http_error_retry"]; + let table = &fixture["strict_tools_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = replay["request"].clone(); + body["thinking"] = case["thinking"].clone(); + body["tools"] = table["tools"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let error_text = case["error_text"].as_str().unwrap(); + let context = MessagesTransformContext { + drops_unsupported_params: case["drops_unsupported_params"].as_bool().unwrap(), + ..MessagesTransformContext::default() + }; + let mut reasons = Vec::new(); + let mut expected = if case["stripped"].as_bool().unwrap() { + reasons.push(RetryReason::ThinkingSignature); + replay["stripped_request"].clone() + } else { + body.clone() + }; + expected["tools"] = if case["strict_dropped"].as_bool().unwrap() { + reasons.push(RetryReason::StrictTools { + dropped: usize::try_from(table["dropped"].as_u64().unwrap()).unwrap(), + }); + table["tools_without_strict"].clone() + } else { + table["tools"].clone() + }; + let retried = should_retry_anthropic_messages_on_http_error( + status, error_text, &body, &context, + ); + let transformed = + transform_anthropic_messages_request_on_http_error(status, error_text, body); + retried != case["retried"].as_bool().unwrap() + || transformed + != RetriedRequest { + reasons, + body: expected, + } + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); + } } diff --git a/litellm-rust/crates/llms/src/base_llm/messages/context.rs b/litellm-rust/crates/llms/src/base_llm/messages/context.rs index 3d8b0f41d8c..a08024bc5e5 100644 --- a/litellm-rust/crates/llms/src/base_llm/messages/context.rs +++ b/litellm-rust/crates/llms/src/base_llm/messages/context.rs @@ -123,6 +123,7 @@ pub struct ThinkingContext { pub struct MessagesTransformContext { pub thinking: ThinkingContext, pub drop_params: bool, + pub drops_unsupported_params: bool, } impl MessagesTransformContext { pub fn new(capabilities: MessagesModelCapabilities, drop_params: bool) -> Self { @@ -142,6 +143,7 @@ impl MessagesTransformContext { keeps_display_updates: false, }, drop_params, + drops_unsupported_params: false, } } diff --git a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs index 3fbf214d733..e72688b7a56 100644 --- a/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/vertex_ai/messages/transformation.rs @@ -712,6 +712,7 @@ mod tests { ..ThinkingContext::default() }, drop_params: false, + drops_unsupported_params: false, }, ); assert_eq!(body["system"], json!("be terse")); @@ -729,6 +730,7 @@ mod tests { ..ThinkingContext::default() }, drop_params: false, + drops_unsupported_params: false, } } diff --git a/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json b/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json index 06fb06f316e..843a1689056 100644 --- a/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json +++ b/litellm-rust/crates/llms/tests/fixtures/anthropic_messages_http_error_retry.json @@ -248,5 +248,376 @@ "stripped": false } ] + }, + "claude_strict_tools_rejection": [ + { + "name": "too_many_strict_tools", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "expected": true + }, + { + "name": "too_many_strict_tools_in_an_error_envelope", + "status": 400, + "error_text": "{\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":\"Too many strict tools (24). The maximum is 20.\"}}", + "expected": true + }, + { + "name": "non_ascii_digits", + "status": 400, + "error_text": "Too many strict tools (٢١)", + "expected": true + }, + { + "name": "optional_parameters", + "status": 400, + "error_text": "Schemas contains too many optional parameters (30)", + "expected": true + }, + { + "name": "union_types", + "status": 400, + "error_text": "Schemas contains too many parameters with union types", + "expected": true + }, + { + "name": "compiled_grammar_too_large", + "status": 400, + "error_text": "The compiled grammar is too large", + "expected": true + }, + { + "name": "compiled_grammar_size", + "status": 400, + "error_text": "Compiled grammar size exceeds the limit", + "expected": true + }, + { + "name": "custom_strict", + "status": 400, + "error_text": "tools.3.custom.strict: Extra inputs are not permitted", + "expected": true + }, + { + "name": "custom", + "status": 400, + "error_text": "tools.0.custom: Extra inputs are not permitted", + "expected": true + }, + { + "name": "grammar_compilation_timed_out", + "status": 400, + "error_text": "Grammar compilation timed out", + "expected": true + }, + { + "name": "no_count", + "status": 400, + "error_text": "tools.0.strict: too many strict tools", + "expected": false + }, + { + "name": "empty_count", + "status": 400, + "error_text": "Too many strict tools ()", + "expected": false + }, + { + "name": "lowercase_grammar_timeout", + "status": 400, + "error_text": "grammar compilation timed out", + "expected": false + }, + { + "name": "custom_without_separator", + "status": 400, + "error_text": "tools.0.custom.strict", + "expected": false + }, + { + "name": "uppercase_schemas", + "status": 400, + "error_text": "SCHEMAS CONTAINS TOO MANY OPTIONAL PARAMETERS", + "expected": false + }, + { + "name": "not_a_400", + "status": 429, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "expected": false + }, + { + "name": "empty", + "status": 400, + "error_text": "", + "expected": false + } + ], + "tools_without_strict": [ + { + "name": "strict_dropped_from_every_tool", + "tools": [ + { + "name": "a", + "strict": true, + "input_schema": {} + }, + { + "name": "b", + "strict": false + }, + { + "name": "c" + } + ], + "expected": { + "tools": [ + { + "name": "a", + "input_schema": {} + }, + { + "name": "b" + }, + { + "name": "c" + } + ], + "dropped_count": 1 + } + }, + { + "name": "only_a_true_strict_counts", + "tools": [ + { + "name": "a", + "strict": 1 + }, + { + "name": "b", + "strict": "true" + } + ], + "expected": null + }, + { + "name": "no_strict", + "tools": [ + { + "name": "a" + } + ], + "expected": null + }, + { + "name": "empty", + "tools": [], + "expected": null + }, + { + "name": "non_object_tool", + "tools": [ + { + "name": "a", + "strict": true + }, + "b" + ], + "expected": null + }, + { + "name": "not_a_list", + "tools": { + "name": "a", + "strict": true + }, + "expected": null + }, + { + "name": "null", + "tools": null, + "expected": null + } + ], + "strict_tools_retry": { + "tools": [ + { + "name": "a", + "input_schema": { + "type": "object" + }, + "strict": true + }, + { + "name": "b", + "input_schema": { + "type": "object" + }, + "strict": false + }, + { + "name": "c", + "input_schema": { + "type": "object" + } + } + ], + "tools_without_strict": [ + { + "name": "a", + "input_schema": { + "type": "object" + } + }, + { + "name": "b", + "input_schema": { + "type": "object" + } + }, + { + "name": "c", + "input_schema": { + "type": "object" + } + } + ], + "dropped": 1, + "cases": [ + { + "name": "strict_rejection_under_drop_params", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": false, + "strict_dropped": true + }, + { + "name": "strict_rejection_without_drop_params", + "status": 400, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": false, + "stripped": false, + "strict_dropped": true + }, + { + "name": "compiled_grammar_under_drop_params", + "status": 400, + "error_text": "The compiled grammar is too large", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": false, + "strict_dropped": true + }, + { + "name": "strict_text_on_a_429", + "status": 429, + "error_text": "Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": false, + "stripped": false, + "strict_dropped": false + }, + { + "name": "unrelated_400_under_drop_params", + "status": 400, + "error_text": "max_tokens: 4096 > 64000", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": false, + "stripped": false, + "strict_dropped": false + }, + { + "name": "thinking_rejection_keeps_strict", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": true, + "strict_dropped": false + }, + { + "name": "thinking_rejection_without_drop_params_keeps_strict", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": true, + "stripped": true, + "strict_dropped": false + }, + { + "name": "one_error_naming_both_without_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "enabled", + "budget_tokens": 2048 + }, + "drops_unsupported_params": false, + "retried": true, + "stripped": true, + "strict_dropped": true + }, + { + "name": "pinned_binding_mismatch_with_a_strict_rejection_under_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "drops_unsupported_params": true, + "retried": true, + "stripped": true, + "strict_dropped": true + }, + { + "name": "pinned_binding_mismatch_with_a_strict_rejection_without_drop_params", + "status": 400, + "error_text": "messages.1.content.0: Invalid `signature` in `thinking` block. The block is bound to a different conversation. Too many strict tools (21). The maximum number of strict tools is 20.", + "thinking": { + "type": "adaptive", + "block_binding": { + "prefix_mismatch_behavior": "error" + } + }, + "drops_unsupported_params": false, + "retried": false, + "stripped": true, + "strict_dropped": true + } + ] } } diff --git a/litellm-rust/crates/llms/tests/messages_http_error_retry.rs b/litellm-rust/crates/llms/tests/messages_http_error_retry.rs index 1a2635f701f..ac8ceac4ac6 100644 --- a/litellm-rust/crates/llms/tests/messages_http_error_retry.rs +++ b/litellm-rust/crates/llms/tests/messages_http_error_retry.rs @@ -70,3 +70,69 @@ fn every_messages_provider_retries_like_the_python_base_config( .collect(); assert_eq!(mismatched, Vec::<&str>::new()); } + +#[rstest] +#[case::anthropic(&ANTHROPIC_MESSAGES_CONFIG)] +#[case::azure_ai(&AZURE_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_invoke(&BEDROCK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::bedrock_claude_platform(&BEDROCK_CLAUDE_PLATFORM_MESSAGES_CONFIG)] +#[case::bedrock_mantle(&BEDROCK_MANTLE_MESSAGES_CONFIG)] +#[case::deepseek(&DEEPSEEK_ANTHROPIC_MESSAGES_CONFIG)] +#[case::edenai(&EDENAI_MESSAGES_CONFIG)] +#[case::minimax(&MINIMAX_MESSAGES_CONFIG)] +#[case::openrouter(&OPENROUTER_MESSAGES_CONFIG)] +#[case::tencent(&TENCENT_MESSAGES_CONFIG)] +#[case::vertex_ai(&VERTEX_ANTHROPIC_MESSAGES_CONFIG)] +fn every_messages_provider_drops_rejected_strict_tools_like_the_python_base_config( + #[case] config: &dyn BaseMessagesConfig, +) { + let fixture: Value = serde_json::from_str(include_str!( + "fixtures/anthropic_messages_http_error_retry.json" + )) + .unwrap(); + let replay = &fixture["http_error_retry"]; + let table = &fixture["strict_tools_retry"]; + let cases = table["cases"].as_array().unwrap(); + assert!(!cases.is_empty()); + let mismatched: Vec<&str> = cases + .iter() + .filter(|case| { + let mut body = replay["request"].clone(); + body["thinking"] = case["thinking"].clone(); + body["tools"] = table["tools"].clone(); + let status = u16::try_from(case["status"].as_u64().unwrap()).unwrap(); + let expected = case["retried"].as_bool().unwrap().then(|| { + let mut reasons = Vec::new(); + let mut expected = if case["stripped"].as_bool().unwrap() { + reasons.push(RetryReason::ThinkingSignature); + replay["stripped_request"].clone() + } else { + body.clone() + }; + expected["tools"] = if case["strict_dropped"].as_bool().unwrap() { + reasons.push(RetryReason::StrictTools { + dropped: usize::try_from(table["dropped"].as_u64().unwrap()).unwrap(), + }); + table["tools_without_strict"].clone() + } else { + table["tools"].clone() + }; + RetriedRequest { + reasons, + body: expected, + } + }); + config.retry_after_http_error( + status, + case["error_text"].as_str().unwrap(), + &body, + &MessagesTransformContext { + drops_unsupported_params: case["drops_unsupported_params"].as_bool().unwrap(), + ..MessagesTransformContext::default() + }, + ) != expected + }) + .map(|case| case["name"].as_str().unwrap()) + .collect(); + assert_eq!(mismatched, Vec::<&str>::new()); +} diff --git a/litellm-rust/crates/router/src/lib.rs b/litellm-rust/crates/router/src/lib.rs index b75c1a14446..6955b8a927a 100644 --- a/litellm-rust/crates/router/src/lib.rs +++ b/litellm-rust/crates/router/src/lib.rs @@ -31,6 +31,8 @@ impl Router { .model_list .iter() .map(|model| { + let drop_params = + drops_params_globally || is_on(model.litellm_params.drop_params.as_ref(), env); ( model.model_name.clone(), Deployment { @@ -45,8 +47,8 @@ impl Router { litellm_params: model.litellm_params.clone(), shaping: MessagesShaping { settings: MessagesSettings { - drop_params: drops_params_globally - || is_on(model.litellm_params.drop_params.as_ref(), env), + drop_params, + drops_unsupported_params: drop_params, ..MessagesSettings::default() }, ..MessagesShaping::default() @@ -113,15 +115,18 @@ mod tests { )) .unwrap(); + let settings = Router::from_config(&config, &|name: &str| { + (name == "DROP_PARAMS_FLAG").then(|| " Yes ".to_owned()) + }) + .get("public-model") + .unwrap() + .shaping + .settings + .clone(); + assert_eq!( - Router::from_config(&config, &|name: &str| (name == "DROP_PARAMS_FLAG") - .then(|| " Yes ".to_owned())) - .get("public-model") - .unwrap() - .shaping - .settings - .drop_params, - expected + (settings.drop_params, settings.drops_unsupported_params), + (expected, expected) ); } } diff --git a/litellm/messages/dispatch.py b/litellm/messages/dispatch.py index 6ce20cfad98..24eb8ff4e31 100644 --- a/litellm/messages/dispatch.py +++ b/litellm/messages/dispatch.py @@ -154,7 +154,6 @@ _ASYNC_FEATURES: Final[detectors.Detectors] = MappingProxyType( Feature.FOUNDRY_BETA_ROW: route_detectors.foundry_beta_row, Feature.WEBSEARCH_INTERCEPTION: route_detectors.websearch_interception, Feature.MESSAGES_INTERCEPTOR: route_detectors.intercepted, - Feature.STRICT_TOOLS_RETRY: route_detectors.strict_tools_retry, } ) diff --git a/litellm/rust_bridge/catalog.py b/litellm/rust_bridge/catalog.py index 96d1774943f..db8748186d8 100644 --- a/litellm/rust_bridge/catalog.py +++ b/litellm/rust_bridge/catalog.py @@ -98,8 +98,6 @@ class Feature(str, Enum): """litellm.include_cost_in_streaming_usage or the x-litellm-include-cost-in-streaming-usage header, where Python adds cost to streamed usage.""" TOOL_CUSTOM_FIELD = "tool_custom_field" """Tools carrying a non-spec custom object, which Python drops or hoists for the provider.""" - STRICT_TOOLS_RETRY = "strict_tools_retry" - """Strict tools under drop_params, which Python sends again without strict if Claude rejects their limits.""" CLIENT_HEADERS = "client_headers" """Proxy-forwarded headers that Python merges into extra_headers.""" DEPLOYMENT_HOOK = "deployment_hook" @@ -183,7 +181,6 @@ MESSAGES_PYTHON_FEATURES: Final = (COMMON_PYTHON_FEATURES - {Feature.TIMEOUT_VAR Feature.CACHE_CONTROL_INJECTION, Feature.STREAM_COST_INJECTION, Feature.TOOL_CUSTOM_FIELD, - Feature.STRICT_TOOLS_RETRY, } RESPONSES_PYTHON_FEATURES: Final = COMMON_PYTHON_FEATURES | { diff --git a/litellm/rust_bridge/host/routes/messages.py b/litellm/rust_bridge/host/routes/messages.py index c6568264edb..90add60d76f 100644 --- a/litellm/rust_bridge/host/routes/messages.py +++ b/litellm/rust_bridge/host/routes/messages.py @@ -15,7 +15,7 @@ from pydantic import TypeAdapter, ValidationError import litellm from litellm.litellm_core_utils import exception_mapping_utils -from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.litellm_core_utils.core_helpers import drops_unsupported_params, normalize_drop_params from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj from litellm.litellm_core_utils.reasoning_effort_capability import deployment_reasoning_effort_levels from litellm.litellm_core_utils.request_timeout_resolver import get_configured_request_timeout @@ -71,6 +71,7 @@ class MessagesSettings: reasoning_auto_summary: bool additional_drop_params: Sequence[str] request_timeout: float | None + drops_unsupported_params: bool def response(value: Mapping[str, object]) -> AnthropicMessagesResponse: @@ -392,5 +393,6 @@ def settings(kwargs: Mapping[str, object]) -> dict[str, object]: reasoning_auto_summary=is_reasoning_auto_summary_enabled(), additional_drop_params=handler.additional_drop_paths(kwargs), request_timeout=get_configured_request_timeout(), + drops_unsupported_params=drops_unsupported_params(kwargs.get("drop_params")), ) ) diff --git a/litellm/rust_bridge/messages/detectors.py b/litellm/rust_bridge/messages/detectors.py index aeb7edbb29e..6124701b2a5 100644 --- a/litellm/rust_bridge/messages/detectors.py +++ b/litellm/rust_bridge/messages/detectors.py @@ -7,7 +7,6 @@ from typing import Final import litellm from litellm.anthropic_beta_headers_manager import is_foundry_api_base from litellm.litellm_core_utils.core_helpers import ( - drops_unsupported_params, proxy_request_headers, stream_cost_injection_requested, ) @@ -47,13 +46,6 @@ def tool_custom_field(request: NativeCall) -> bool: return any("custom" in (optional_mapping(tool) or {}) for tool in tools) -def strict_tools_retry(request: NativeCall) -> bool: - tools: Final = optional_sequence(request.resolved.get("tools")) or () - return drops_unsupported_params(request.resolved.get("drop_params")) and any( - (optional_mapping(tool) or {}).get("strict") is True for tool in tools - ) - - def stream_cost_injection(request: NativeCall) -> bool: """Python adds response cost to streaming usage when the global setting or the request header asks for it.""" return streaming(request) and stream_cost_injection_requested(proxy_request_headers(request.resolved)) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py index fc689e2723c..3049801b555 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/messages/test_native_parity.py @@ -373,7 +373,7 @@ KNOWN_GAPS: Final[Mapping[str, Mapping[str, str | None]]] = MappingProxyType( logged_request=F_MOCK, ), "cache_control_injection_points": _gaps(request=F_CACHE_INJECTION, logged_request=F_CACHE_INJECTION), - "tool_custom_field": _gaps(request=F_TOOL_CUSTOM), + "tool_custom_field": _gaps(request=F_TOOL_CUSTOM, logged_request=F_TOOL_CUSTOM), "stream_cost_injection": _gaps(outcome=F_STREAM_COST), "enable_prompt_caching": _gaps(request=F_CACHE_INJECTION, logged_request=F_CACHE_INJECTION), } @@ -730,14 +730,6 @@ FALLBACK_CASES: Final = ( _kw(system="Answer in one word.", cache_control_injection_points=[{"location": "message", "role": "system"}]), ), Case("tool_custom_field", _kw(tools=[{**WEATHER_TOOL, "custom": {"defer_loading": True}}])), - Case( - "strict_tools_rejected_then_retried", - _kw(tools=[{**WEATHER_TOOL, "strict": True}], drop_params=True), - upstream=( - _failure(400, "invalid_request_error", "tools.0.strict: too many strict tools"), - _ok(), - ), - ), Case("modify_params", _kw(max_tokens=1), settings=MappingProxyType({"modify_params": True})), Case( "stream_cost_injection", @@ -828,6 +820,97 @@ THINKING_RETRY_CASES: Final = ( ), ) +STRICT_REJECTION: Final = "Too many strict tools (21). The maximum number of strict tools is 20." +STRICT_TOOLS: Final[tuple[JsonValue, ...]] = ( + {**WEATHER_TOOL, "strict": True}, + {**WEATHER_TOOL, "name": "get_time", "strict": False}, + dict(WEATHER_TOOL, name="get_tide"), +) + + +def _strict(**kwargs: object) -> Mapping[str, object]: + return _kw(tools=list(STRICT_TOOLS), **kwargs) + + +STRICT_TOOLS_RETRY_CASES: Final = ( + Case( + "strict_tools_rejected_then_retried", _strict(drop_params=True), upstream=(_rejected(STRICT_REJECTION), _ok()) + ), + Case( + "strict_tools_rejected_then_retried_stream", + _strict(drop_params=True, stream=True), + upstream=(_rejected(STRICT_REJECTION), _stream(TEXT_STREAM)), + ), + Case( + "strict_tools_rejected_then_retried_under_global_drop_params", + _strict(), + upstream=(_rejected("The compiled grammar is too large"), _ok()), + settings=MappingProxyType({"drop_params": True}), + ), + Case( + "strict_tools_custom_field_rejected_then_retried", + _strict(drop_params="true"), + upstream=(_rejected("tools.0.custom.strict: Extra inputs are not permitted"), _ok()), + ), + Case( + "strict_tools_rejected_without_drop_params", + _strict(), + upstream=(_rejected(STRICT_REJECTION),), + settings=MappingProxyType({"drop_params": False}), + ), + Case( + "strict_tools_rejected_under_a_truthy_global_drop_params_that_is_not_true", + _strict(), + upstream=(_rejected(STRICT_REJECTION),), + settings=MappingProxyType({"drop_params": 1}), + ), + Case( + "strict_rejection_without_a_strict_tool", + _kw(tools=[{**WEATHER_TOOL, "strict": False}], drop_params=True), + upstream=(_rejected(STRICT_REJECTION),), + ), + Case( + "strict_rejection_text_on_a_429_is_not_retried", + _strict(drop_params=True), + upstream=(_rejected(STRICT_REJECTION, status=429),), + ), + Case( + "strict_tools_rejected_twice", + _strict(drop_params=True), + upstream=(_rejected(STRICT_REJECTION), _rejected("Grammar compilation timed out")), + ), + Case( + "strict_tools_rejected_twice_stream", + _strict(drop_params=True, stream=True), + upstream=(_rejected(STRICT_REJECTION), _rejected(STRICT_REJECTION)), + ), + Case( + "strict_tools_and_thinking_strict_rejected_then_retried", + _replay(tools=list(STRICT_TOOLS), drop_params=True), + upstream=(_rejected(STRICT_REJECTION), _ok()), + ), + Case( + "strict_tools_and_thinking_signature_rejected_then_retried", + _replay(tools=list(STRICT_TOOLS), drop_params=True), + upstream=(_rejected(), _ok()), + ), + Case( + "strict_tools_and_thinking_each_rejected_spends_the_one_retry", + _replay(tools=list(STRICT_TOOLS), drop_params=True), + upstream=(_rejected(STRICT_REJECTION), _rejected()), + ), + Case( + "strict_tools_and_thinking_signature_then_strict_rejected", + _replay(tools=list(STRICT_TOOLS), drop_params=True), + upstream=(_rejected(), _rejected(STRICT_REJECTION), _ok()), + ), + Case( + "one_error_naming_both_drops_thinking_and_strict_without_drop_params", + _replay(tools=list(STRICT_TOOLS)), + upstream=(_rejected(f"{INVALID_SIGNATURE}. {STRICT_REJECTION}"), _ok()), + ), +) + CASES: Final = ( *REQUEST_CASES, *RESPONSE_CASES, @@ -835,6 +918,7 @@ CASES: Final = ( *STREAM_CASES, *SYNC_CASES, *THINKING_RETRY_CASES, + *STRICT_TOOLS_RETRY_CASES, *FALLBACK_CASES, ) @@ -1066,6 +1150,7 @@ def _logged(recorder: RecordingLogger, events: tuple[str, ...]) -> tuple[Mapping "logged_request": { "messages": details.get("messages"), "thinking": details.get("thinking"), + "tools": details.get("tools"), "complete_input_dict": _mapping(details.get("additional_args")).get("complete_input_dict"), }, } diff --git a/tests/test_litellm_rust/messages/test_dispatch.py b/tests/test_litellm_rust/messages/test_dispatch.py index 65c929ce6a9..0968ef22e34 100644 --- a/tests/test_litellm_rust/messages/test_dispatch.py +++ b/tests/test_litellm_rust/messages/test_dispatch.py @@ -614,27 +614,24 @@ async def test_replayed_thinking_is_served_natively_under_the_shipped_rules( @pytest.mark.asyncio @pytest.mark.parametrize( - "kwargs", + ("kwargs", "global_drop_params"), ( - pytest.param({"tools": [STRICT_TOOL]}, id="drop-params-unset"), - pytest.param({"drop_params": False, "tools": [STRICT_TOOL]}, id="drop-params-off"), - pytest.param( - {"drop_params": True, "tools": [{"name": "a", "input_schema": {"type": "object"}}]}, id="no-strict" - ), + pytest.param({"drop_params": True, "tools": [STRICT_TOOL]}, False, id="drop-params-kwarg"), + pytest.param({"tools": [STRICT_TOOL]}, True, id="drop-params-global"), + pytest.param({"tools": [STRICT_TOOL]}, False, id="drop-params-unset"), ), ) -async def test_strict_tools_stay_native_unless_python_could_drop_strict( - monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object] +async def test_strict_tools_are_served_natively_under_the_shipped_rules( + monkeypatch: pytest.MonkeyPatch, kwargs: Mapping[str, object], global_drop_params: bool ) -> None: - rules: Final[Rules] = ( - RouteRule(Route.MESSAGES, Rollout.PYTHON_ONLY, features=frozenset({Feature.STRICT_TOOLS_RETRY})), - RouteRule(Route.MESSAGES, Rollout.RUST_REQUIRED), - ) + monkeypatch.setenv("LITELLM_RUST", "1") + monkeypatch.setattr(litellm, "callbacks", []) + monkeypatch.setattr(litellm, "cache", None) + monkeypatch.setattr(litellm, "drop_params", global_drop_params) native_result: Final = response() - monkeypatch.setattr(litellm, "drop_params", False) async def python(*call_args: object, **python_kwargs: object) -> AnthropicMessagesResponse: # kwargs-ok: stub - pytest.fail("Python cannot drop strict here") + pytest.fail("strict tools stay on the native path") async def native(request: NativeCall) -> AnthropicMessagesResponse: return native_result @@ -645,7 +642,7 @@ async def test_strict_tools_stay_native_unless_python_could_drop_strict( python=python, binding=amessages_binding(native), native=lambda hook, request, call_args, native_kwargs: hook(request), - rules=rules, + rules=RULES, ) assert served is native_result @@ -668,12 +665,6 @@ ASYNC_ONLY_FEATURES: Final = ( {}, id="messages-interceptor", ), - pytest.param( - Feature.STRICT_TOOLS_RETRY, {"drop_params": True, "tools": [STRICT_TOOL]}, {}, id="strict-tools-retry" - ), - pytest.param( - Feature.STRICT_TOOLS_RETRY, {"tools": [STRICT_TOOL]}, {"drop_params": True}, id="strict-tools-retry-global" - ), ) diff --git a/tests/unit/llms/anthropic/test_messages_http_error_retry_cases.py b/tests/unit/llms/anthropic/test_messages_http_error_retry_cases.py index 58f7012921f..a9edb2160f7 100644 --- a/tests/unit/llms/anthropic/test_messages_http_error_retry_cases.py +++ b/tests/unit/llms/anthropic/test_messages_http_error_retry_cases.py @@ -8,9 +8,13 @@ import pytest from pydantic import TypeAdapter from typing_extensions import ReadOnly, TypedDict +import litellm from litellm.llms.anthropic.common_utils import ( + ToolsWithoutStrict, is_anthropic_invalid_thinking_block_error, is_anthropic_thinking_binding_mismatch_error, + is_claude_strict_tools_rejection, + tools_without_strict, ) from litellm.llms.anthropic.messages.native.transformation import AnthropicMessagesConfig from litellm.llms.deepseek.messages.transformation import DeepSeekAnthropicMessagesConfig @@ -42,14 +46,54 @@ class _RetryTable(TypedDict): cases: ReadOnly[tuple[_RetryCase, ...]] +class _StrictRejectionCase(TypedDict): + name: ReadOnly[str] + status: ReadOnly[int] + error_text: ReadOnly[str] + expected: ReadOnly[bool] + + +class _ToolsWithoutStrictExpected(TypedDict): + tools: ReadOnly[tuple[Mapping[str, object], ...]] + dropped_count: ReadOnly[int] + + +class _ToolsWithoutStrictCase(TypedDict): + name: ReadOnly[str] + tools: ReadOnly[object] + expected: ReadOnly[_ToolsWithoutStrictExpected | None] + + +class _StrictRetryCase(TypedDict): + name: ReadOnly[str] + status: ReadOnly[int] + error_text: ReadOnly[str] + thinking: ReadOnly[Mapping[str, object]] + drops_unsupported_params: ReadOnly[bool] + retried: ReadOnly[bool] + stripped: ReadOnly[bool] + strict_dropped: ReadOnly[bool] + + +class _StrictRetryTable(TypedDict): + tools: ReadOnly[tuple[Mapping[str, object], ...]] + tools_without_strict: ReadOnly[tuple[Mapping[str, object], ...]] + dropped: ReadOnly[int] + cases: ReadOnly[tuple[_StrictRetryCase, ...]] + + class _Fixture(TypedDict): invalid_thinking_block_error: ReadOnly[tuple[_TextCase, ...]] thinking_binding_mismatch_error: ReadOnly[tuple[_TextCase, ...]] http_error_retry: ReadOnly[_RetryTable] + claude_strict_tools_rejection: ReadOnly[tuple[_StrictRejectionCase, ...]] + tools_without_strict: ReadOnly[tuple[_ToolsWithoutStrictCase, ...]] + strict_tools_retry: ReadOnly[_StrictRetryTable] FIXTURE: Final = TypeAdapter(_Fixture).validate_json(FIXTURE_PATH.read_text()) RETRY: Final = FIXTURE["http_error_retry"] +STRICT_RETRY: Final = FIXTURE["strict_tools_retry"] @pytest.mark.parametrize( @@ -66,7 +110,25 @@ def test_binding_mismatch_errors_match_the_shared_cases(case: _TextCase) -> None assert is_anthropic_thinking_binding_mismatch_error(case["error_text"]) is case["expected"] -def _rejection(case: _RetryCase) -> httpx.HTTPStatusError: +@pytest.mark.parametrize( + "case", [pytest.param(case, id=case["name"]) for case in FIXTURE["claude_strict_tools_rejection"]] +) +def test_strict_tools_rejections_match_the_shared_cases(case: _StrictRejectionCase) -> None: + assert is_claude_strict_tools_rejection(case["status"], case["error_text"]) is case["expected"] + + +@pytest.mark.parametrize("case", [pytest.param(case, id=case["name"]) for case in FIXTURE["tools_without_strict"]]) +def test_tools_without_strict_matches_the_shared_cases(case: _ToolsWithoutStrictCase) -> None: + expected: Final = case["expected"] + + assert tools_without_strict(copy.deepcopy(case["tools"])) == ( + None + if expected is None + else ToolsWithoutStrict(tools=tuple(expected["tools"]), dropped_count=expected["dropped_count"]) + ) + + +def _rejection(case: _RetryCase | _StrictRetryCase) -> httpx.HTTPStatusError: request: Final = httpx.Request("POST", "https://api.anthropic.com/v1/messages") return httpx.HTTPStatusError( "rejected", @@ -93,3 +155,34 @@ def test_the_http_error_retry_matches_the_shared_cases(config: BaseAnthropicMess assert retried is case["retried"] assert transformed == (RETRY["stripped_request"] if case["stripped"] else body) + + +@pytest.mark.parametrize( + "config", + [AnthropicMessagesConfig(), DeepSeekAnthropicMessagesConfig()], + ids=["anthropic", "deepseek"], +) +@pytest.mark.parametrize("case", [pytest.param(case, id=case["name"]) for case in STRICT_RETRY["cases"]]) +def test_the_strict_tools_retry_matches_the_shared_cases( + monkeypatch: pytest.MonkeyPatch, config: BaseAnthropicMessagesConfig, case: _StrictRetryCase +) -> None: + monkeypatch.setattr(litellm, "drop_params", False) + body: Final = { + **copy.deepcopy(RETRY["request"]), + "thinking": case["thinking"], + "tools": copy.deepcopy(list(STRICT_RETRY["tools"])), + } + expected: Final = { + **copy.deepcopy(RETRY["stripped_request"] if case["stripped"] else body), + "tools": list(STRICT_RETRY["tools_without_strict"] if case["strict_dropped"] else STRICT_RETRY["tools"]), + } + + retried: Final = config.should_retry_anthropic_messages_on_http_error( + e=_rejection(case), litellm_params={"drop_params": case["drops_unsupported_params"]}, request_body=body + ) + transformed: Final = config.transform_anthropic_messages_request_on_http_error( + e=_rejection(case), request_data=copy.deepcopy(body) + ) + + assert retried is case["retried"] + assert transformed == expected diff --git a/tests/unit/rust_bridge/host/routes/test_messages.py b/tests/unit/rust_bridge/host/routes/test_messages.py index aa7dcc74841..8ed4152a3e1 100644 --- a/tests/unit/rust_bridge/host/routes/test_messages.py +++ b/tests/unit/rust_bridge/host/routes/test_messages.py @@ -1,3 +1,4 @@ +from collections.abc import Mapping from types import MappingProxyType from typing import Final @@ -43,9 +44,36 @@ def test_settings_project_caller_configuration_without_resolving_a_model(monkeyp "reasoning_auto_summary": True, "additional_drop_params": ("metadata.user_id",), "request_timeout": 1800.0, + "drops_unsupported_params": True, } +@pytest.mark.parametrize( + ("global_drop_params", "kwargs", "drop_params", "drops_unsupported_params"), + [ + pytest.param(True, {}, True, True, id="global-true"), + pytest.param(1, {}, True, False, id="global-truthy-non-bool"), + pytest.param(False, {"drop_params": "True"}, True, True, id="kwarg-flag-string"), + pytest.param(False, {"drop_params": 1}, True, True, id="kwarg-one"), + pytest.param(False, {"drop_params": "maybe"}, False, False, id="kwarg-not-a-flag"), + pytest.param(False, {}, False, False, id="unset"), + ], +) +def test_strict_tools_retry_reads_drop_params_as_the_python_retry_check_does( + monkeypatch: pytest.MonkeyPatch, + global_drop_params: object, + kwargs: Mapping[str, object], + drop_params: bool, + drops_unsupported_params: bool, +) -> None: + monkeypatch.setattr(litellm, "drop_params", global_drop_params) + + projected: Final = messages_host.settings(kwargs) + + assert projected["drop_params"] is drop_params + assert projected["drops_unsupported_params"] is drops_unsupported_params + + @pytest.mark.parametrize( ("timeout", "explicitly_set", "expected"), [ diff --git a/tests/unit/rust_bridge/test_detectors.py b/tests/unit/rust_bridge/test_detectors.py index 010427cc9b9..bab41648eb6 100644 --- a/tests/unit/rust_bridge/test_detectors.py +++ b/tests/unit/rust_bridge/test_detectors.py @@ -31,7 +31,6 @@ MAPPED_FILE: Final = { } MAPPED_FILE_INPUT: Final = [{"role": "user", "content": [{"type": "input_file", "file_id": "litellm-file"}]}] CACHED_TEXT: Final = {"type": "input_text", "text": "hi", "cache_control": {"type": "ephemeral"}} -STRICT_TOOL: Final = {"name": "b", "input_schema": {"type": "object"}, "strict": True} TOOL_SEARCH_TYPE: Final = sorted(ANTHROPIC_TOOL_SEARCH_TOOL_TYPES)[0] @@ -197,21 +196,6 @@ def _wrap_nested_level(value: object, depth: int) -> object: (messages_detectors.tool_custom_field, {"tools": [{"name": "a"}, {"name": "b", "custom": {}}]}, True), (messages_detectors.tool_custom_field, {"tools": [{"name": "a", "defer_loading": True}]}, False), (messages_detectors.tool_custom_field, {}, False), - (messages_detectors.strict_tools_retry, {"drop_params": True, "tools": [{"name": "a"}, STRICT_TOOL]}, True), - ( - messages_detectors.strict_tools_retry, - {"drop_params": True, "tools": [{**STRICT_TOOL, "strict": False}]}, - False, - ), - ( - messages_detectors.strict_tools_retry, - {"additional_drop_params": ["metadata.user_id"], "tools": [STRICT_TOOL]}, - False, - ), - (messages_detectors.strict_tools_retry, {"tools": [STRICT_TOOL]}, False), - (messages_detectors.strict_tools_retry, {"drop_params": False, "tools": [STRICT_TOOL]}, False), - (messages_detectors.strict_tools_retry, {"drop_params": True, "tools": [{"name": "a"}]}, False), - (messages_detectors.strict_tools_retry, {"drop_params": True}, False), (detectors.drop_params, {"drop_params": True}, True), (detectors.drop_params, {"drop_params": False}, False), (detectors.drop_params, {"drop_params": " True "}, True),