From bdf854c3ea3ecbfd399c26a33710d2c9644eb616 Mon Sep 17 00:00:00 2001 From: "devin-ai-integration[bot]" <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Thu, 24 Sep 2026 19:08:32 +0000 Subject: [PATCH] feat(rust): shape Anthropic Messages requests natively (#42982) * test(rust): encode anthropic response serialization shape as rstest cases Co-Authored-By: Claude Opus 5.5 * feat(rust): shape Anthropic Messages requests natively The Rust Messages route only relayed the body. It now runs the request shaping the Python handler does for the direct Anthropic provider: history sanitizers (empty blocks, tool ids, replayed web search results, provider_specific_fields, encrypted reasoning, advisor blocks), reasoning_effort and adaptive/legacy thinking translation against the model's capability flags, the sampling and speed gates under drop_params, the metadata allowlist, additional_drop_params, reasoning auto summary, OAuth and ANTHROPIC_AUTH_TOKEN credentials, provider_specific_header merging and anthropic-beta injection. Capability flags and LiteLLM settings reach Rust through route_host.shaping(). A request the route rejects before the call now maps to BadRequestError instead of APIConnectionError Co-Authored-By: Claude Fable 5.1 * test(rust): port Anthropic Messages shaping tests and pin comment contracts as cases Every Python unit test that exercises the ported shaping for the direct Anthropic provider now has a named rstest counterpart, and every comment that stated a behavior contract is deleted in favor of a case that pins it. Measured with cargo-mutants over the touched files, all viable mutants are caught Porting the tests surfaced parity gaps, fixed here to match Python: every casing of a forwarded anthropic-beta header is merged, replayed web search results are rewritten from their own block (an empty result keeps its slot and a server_tool_use with a non-string query stays), an empty output_config.effort falls back to medium, speed and reasoning effort errors quote values the way Python does, additional_drop_params apply after metadata validation and the auto summary and never touch model or messages, and a non-string metadata.user_id is rejected before the call * fix(rust): resolve Messages credentials through the secret source and scope headers by resolved provider The native Messages route read ANTHROPIC_API_KEY, ANTHROPIC_AUTH_TOKEN and the base URL straight from the process environment, so a key or base held in a configured secret manager was never found. Each provider config now declares its secret names and the route resolves them through the same SecretSource the OCR route uses, with the Python bridge passing in litellm's configured manager provider_specific_header entries were scoped by the explicit custom_llm_provider only, falling back to anthropic, so an azure_ai/ model lost its azure_ai scoped headers. Scoping now happens in the route after the provider is resolved from the model, as Python's handler does The Azure config now adds the same anthropic-beta feature headers Python's Azure route adds, and the metadata allowlist, reasoning auto summary and history sanitizers move from the core route into the llms crate, mirroring their home in Python's messages handler * test(rust): escape the dot in the metadata.user_id match pattern * refactor(rust-bridge): project Messages capabilities without mutable dicts The capability flags and effort tiers were built as dict comprehensions, which the type-discipline gate counts as mutable construction, and the asdict call carried a mutable-ok suppression that suppressed nothing. The flags are now passed one by one and the effort tiers are a frozen dataclass, which asdict projects to the same map the native side reads --------- Co-authored-by: Yujong Lee Co-authored-by: Claude Opus 5.5 --- litellm-rust/Cargo.lock | 1 + .../core-utils/src/dot_notation_indexing.rs | 274 +++ .../src/get_provider_specific_headers.rs | 93 + litellm-rust/crates/core-utils/src/lib.rs | 2 + .../crates/core/src/messages/common_utils.rs | 2 +- .../crates/core/src/messages/error.rs | 28 + litellm-rust/crates/core/src/messages/mod.rs | 8 +- .../crates/core/src/messages/prepare.rs | 501 +++++- .../crates/core/src/messages/route.rs | 52 +- .../crates/core/src/messages/tests.rs | 149 +- .../crates/core/src/messages/types.rs | 104 +- .../crates/llms/src/anthropic/common_utils.rs | 1568 +++++++++++++++++ .../messages/handler.rs | 270 +++ .../messages/headers.rs | 643 +++++++ .../experimental_pass_through/messages/mod.rs | 3 + .../messages/thinking.rs | 1182 +++++++++++++ .../messages/transformation.rs | 808 ++++++++- litellm-rust/crates/llms/src/anthropic/mod.rs | 1 + .../anthropic/messages_transformation.rs | 145 +- .../anthropic_messages/transformation.rs | 232 ++- .../python-bridge/src/routes/messages/host.rs | 175 +- .../python-bridge/src/routes/messages/mod.rs | 3 +- litellm-rust/crates/types/Cargo.toml | 3 + .../anthropic_messages/anthropic_request.rs | 156 ++ .../anthropic_messages/anthropic_response.rs | 60 +- litellm-rust/crates/types/src/utils.rs | 15 + litellm/rust_bridge/messages/route_host.py | 110 +- tests/test_litellm/rust_bridge/AGENTS.md | 2 + .../rust_bridge/messages/test_route_host.py | 112 ++ .../rust_bridge/messages/test_secrets.py | 111 ++ .../messages/test_request_shaping.py | 237 +++ 31 files changed, 6891 insertions(+), 159 deletions(-) create mode 100644 litellm-rust/crates/core-utils/src/dot_notation_indexing.rs create mode 100644 litellm-rust/crates/core-utils/src/get_provider_specific_headers.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/common_utils.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/handler.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/headers.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/thinking.rs create mode 100644 tests/test_litellm/rust_bridge/messages/test_route_host.py create mode 100644 tests/test_litellm/rust_bridge/messages/test_secrets.py create mode 100644 tests/test_litellm_rust/messages/test_request_shaping.py diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index ef032bfe55c..0a91f0759c2 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3414,6 +3414,7 @@ dependencies = [ name = "litellm-types" version = "0.1.0" dependencies = [ + "rstest", "serde", "serde_json", ] diff --git a/litellm-rust/crates/core-utils/src/dot_notation_indexing.rs b/litellm-rust/crates/core-utils/src/dot_notation_indexing.rs new file mode 100644 index 00000000000..be81d900f57 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/dot_notation_indexing.rs @@ -0,0 +1,274 @@ +use serde_json::Value; + +#[derive(Clone, Debug, PartialEq, Eq)] +enum Segment { + Field(String), + Every, + Index(usize), +} + +fn parse_segments(path: &str) -> Option> { + let mut segments = Vec::new(); + let mut rest = path; + while !rest.is_empty() { + if let Some(after_open) = rest.strip_prefix('[') { + let (inside, after) = after_open.split_once(']')?; + segments.push(match inside { + "*" => Segment::Every, + index => Segment::Index(index.trim().parse().ok()?), + }); + rest = after.strip_prefix('.').unwrap_or(after); + continue; + } + let end = rest.find(['.', '[']).unwrap_or(rest.len()); + let (field, after) = rest.split_at(end); + if !field.is_empty() { + segments.push(Segment::Field(field.to_string())); + } + rest = after.strip_prefix('.').unwrap_or(after); + } + Some(segments) +} + +fn without_path(value: Value, segments: &[Segment]) -> Value { + let Some((segment, tail)) = segments.split_first() else { + return value; + }; + match (segment, value) { + (Segment::Field(name), Value::Object(object)) => Value::Object( + object + .into_iter() + .filter_map(|(key, item)| { + if key != *name { + return Some((key, item)); + } + (!tail.is_empty()).then(|| (key, without_path(item, tail))) + }) + .collect(), + ), + (Segment::Every, Value::Array(items)) => Value::Array( + items + .into_iter() + .map(|item| without_path(item, tail)) + .collect(), + ), + (Segment::Index(index), Value::Array(items)) => Value::Array( + items + .into_iter() + .enumerate() + .map(|(position, item)| { + if position == *index { + without_path(item, tail) + } else { + item + } + }) + .collect(), + ), + (_, value) => value, + } +} + +pub fn delete_nested_value(value: Value, path: &str) -> Value { + match parse_segments(path) { + Some(segments) => without_path(value, &segments), + None => value, + } +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + use serde_json::json; + + use super::*; + + #[fixture] + fn body() -> Value { + json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }) + } + + #[rstest] + #[case::top_level_field("top", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}} + }))] + #[case::whole_object("meta", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "top": 0.7 + }))] + #[case::nested_field("meta.inner.drop", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"keep": 2}}, + "top": 0.7 + }))] + #[case::trailing_dot("meta.inner.drop.", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"keep": 2}}, + "top": 0.7 + }))] + #[case::leading_and_doubled_dots(".meta..inner.drop", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"keep": 2}}, + "top": 0.7 + }))] + #[case::field_in_every_element("tools[*].examples", json!({ + "tools": [ + {"name": "t0", "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }))] + #[case::whole_array_field_in_every_element("tools[*].arr", json!({ + "tools": [ + {"name": "t0", "examples": ["a"]}, + {"name": "t1", "examples": ["b"]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }))] + #[case::field_in_indexed_element("tools[1].examples", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }))] + #[case::padded_index("tools[ 1 ].examples", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }))] + #[case::field_right_after_bracket("tools[0]examples", json!({ + "tools": [ + {"name": "t0", "arr": [{"f": 1, "k": 1}, {"f": 2, "k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }))] + #[case::index_then_wildcard("tools[0].arr[*].f", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"k": 1}, {"k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }))] + #[case::wildcard_then_index_only_where_it_exists("tools[*].arr[1].f", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"f": 1, "k": 1}, {"k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"f": 3, "k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }))] + #[case::nested_wildcards("tools[*].arr[*].f", json!({ + "tools": [ + {"name": "t0", "examples": ["a"], "arr": [{"k": 1}, {"k": 2}]}, + {"name": "t1", "examples": ["b"], "arr": [{"k": 3}]} + ], + "meta": {"user": "u", "inner": {"drop": 1, "keep": 2}}, + "top": 0.7 + }))] + fn deletes_the_addressed_field(body: Value, #[case] path: &str, #[case] expected: Value) { + assert_eq!(delete_nested_value(body, path), expected); + } + + #[rstest] + #[case::empty_path("")] + #[case::missing_field("missing")] + #[case::missing_parent("missing.field")] + #[case::field_through_a_scalar("top.value")] + #[case::field_on_an_array("tools.name")] + #[case::index_on_an_object("meta[0].user")] + #[case::wildcard_on_an_object("meta[*].user")] + #[case::wildcard_over_scalars("tools[*].examples[*].name")] + #[case::index_out_of_range("tools[5].name")] + #[case::every_element_itself("tools[*]")] + #[case::indexed_element_itself("tools[0]")] + #[case::nested_element_itself("tools[*].arr[0]")] + #[case::negative_index("tools[-1].name")] + #[case::non_numeric_index("tools[x].name")] + #[case::empty_index("tools[].name")] + #[case::unclosed_bracket("top[0")] + fn leaves_the_value_untouched(body: Value, #[case] path: &str) { + assert_eq!(delete_nested_value(body.clone(), path), body); + } + + #[rstest] + #[case::wildcards_indices_and_nesting( + json!({"tools": [ + {"name": "t0", "configs": [{"id": "c0", "remove_me": 1, "keep": 1}, {"id": "c1", "remove_me": 2, "keep": 2}], "metadata": {"drop_this": 1, "preserve": 1}}, + {"name": "t1", "configs": [{"id": "c0", "remove_me": 3, "keep": 3}, {"id": "c1", "remove_me": 4, "keep": 4}], "metadata": {"drop_this": 2, "preserve": 2}}, + {"name": "t2", "configs": [{"id": "c0", "remove_me": 5, "keep": 5}], "metadata": {"drop_this": 3, "preserve": 3}} + ]}), + &["tools[*].configs[1].remove_me", "tools[1].metadata.drop_this", "tools[*].configs[*].id"], + json!({"tools": [ + {"name": "t0", "configs": [{"remove_me": 1, "keep": 1}, {"keep": 2}], "metadata": {"drop_this": 1, "preserve": 1}}, + {"name": "t1", "configs": [{"remove_me": 3, "keep": 3}, {"keep": 4}], "metadata": {"preserve": 2}}, + {"name": "t2", "configs": [{"remove_me": 5, "keep": 5}], "metadata": {"drop_this": 3, "preserve": 3}} + ]}), + )] + #[case::simple_and_wildcard_nesting( + json!({ + "tools": [{"name": "t1", "simple_nested": {"remove": 1, "keep": 2}, "complex": [{"nested": {"remove": 3, "keep": 4}}]}], + "top_level_remove": "should_go", + "top_level_keep": "should_stay" + }), + &["tools[*].simple_nested.remove", "tools[*].complex[*].nested.remove"], + json!({ + "tools": [{"name": "t1", "simple_nested": {"keep": 2}, "complex": [{"nested": {"keep": 4}}]}], + "top_level_remove": "should_go", + "top_level_keep": "should_stay" + }), + )] + #[case::triple_nested_wildcards( + json!({"tools": [{"name": "t1", "arr1": [ + {"arr2": [{"field": 1, "keep": 1}, {"field": 2, "keep": 2}]}, + {"arr2": [{"field": 3, "keep": 3}]} + ]}]}), + &["tools[*].arr1[*].arr2[*].field"], + json!({"tools": [{"name": "t1", "arr1": [ + {"arr2": [{"keep": 1}, {"keep": 2}]}, + {"arr2": [{"keep": 3}]} + ]}]}), + )] + fn applies_paths_in_sequence( + #[case] value: Value, + #[case] paths: &[&str], + #[case] expected: Value, + ) { + let deleted = paths + .iter() + .fold(value, |value, path| delete_nested_value(value, path)); + assert_eq!(deleted, expected); + } +} diff --git a/litellm-rust/crates/core-utils/src/get_provider_specific_headers.rs b/litellm-rust/crates/core-utils/src/get_provider_specific_headers.rs new file mode 100644 index 00000000000..bfcd448e2d8 --- /dev/null +++ b/litellm-rust/crates/core-utils/src/get_provider_specific_headers.rs @@ -0,0 +1,93 @@ +use litellm_types::utils::{ProviderSpecificHeader, ProviderSpecificHeaders}; +use serde_json::{Map, Value}; + +pub fn get_provider_specific_headers( + provider_specific_header: Option<&ProviderSpecificHeaders>, + custom_llm_provider: &str, +) -> Map { + let entries: &[ProviderSpecificHeader] = match provider_specific_header { + None => &[], + Some(ProviderSpecificHeaders::One(entry)) => std::slice::from_ref(entry), + Some(ProviderSpecificHeaders::Many(entries)) => entries, + }; + entries + .iter() + .filter(|entry| { + entry + .custom_llm_provider + .split(',') + .any(|scoped| scoped.trim() == custom_llm_provider) + }) + .flat_map(|entry| entry.extra_headers.clone()) + .collect() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::single_entry_for_the_provider( + json!({"custom_llm_provider": "anthropic", "extra_headers": {"Authorization": "Bearer t", "Custom-Header": "v"}}), + json!({"Authorization": "Bearer t", "Custom-Header": "v"}), + )] + #[case::single_entry_for_another_provider( + json!({"custom_llm_provider": "openai", "extra_headers": {"Authorization": "Bearer t"}}), + json!({}), + )] + #[case::provider_in_a_comma_separated_scope( + json!({"custom_llm_provider": "bedrock,anthropic,vertex_ai", "extra_headers": {"anthropic-beta": "context-1m-2025-08-07"}}), + json!({"anthropic-beta": "context-1m-2025-08-07"}), + )] + #[case::provider_missing_from_a_comma_separated_scope( + json!({"custom_llm_provider": "bedrock,vertex_ai", "extra_headers": {"anthropic-beta": "test"}}), + json!({}), + )] + #[case::scope_with_spaces( + json!({"custom_llm_provider": "bedrock, anthropic , vertex_ai", "extra_headers": {"anthropic-beta": "test"}}), + json!({"anthropic-beta": "test"}), + )] + #[case::scope_names_must_match_exactly( + json!({"custom_llm_provider": "anthropic_text", "extra_headers": {"anthropic-beta": "test"}}), + json!({}), + )] + #[case::entries_scope_independently( + json!([ + {"custom_llm_provider": "anthropic,bedrock,vertex_ai", "extra_headers": {"anthropic-beta": "context-1m-2025-08-07"}}, + {"custom_llm_provider": "bedrock", "extra_headers": {"x-bedrock-only": "no"}}, + {"custom_llm_provider": "anthropic", "extra_headers": {"authorization": "Bearer sk-ant-oat01-fake-token"}} + ]), + json!({"anthropic-beta": "context-1m-2025-08-07", "authorization": "Bearer sk-ant-oat01-fake-token"}), + )] + #[case::later_entries_win( + json!([ + {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "first"}}, + {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "second"}} + ]), + json!({"x-scoped": "second"}), + )] + #[case::empty_list(json!([]), json!({}))] + #[case::entry_without_scope(json!({"extra_headers": {"x-scoped": "yes"}}), json!({}))] + #[case::entry_without_headers(json!({"custom_llm_provider": "anthropic"}), json!({}))] + fn provider_specific_headers_match_the_scoped_provider( + #[case] configured: Value, + #[case] expected: Value, + ) { + let configured: ProviderSpecificHeaders = serde_json::from_value(configured).unwrap(); + assert_eq!( + Value::Object(get_provider_specific_headers( + Some(&configured), + "anthropic" + )), + expected + ); + } + + #[test] + fn no_configured_headers_match_nothing() { + assert_eq!(get_provider_specific_headers(None, "anthropic"), Map::new()); + } +} diff --git a/litellm-rust/crates/core-utils/src/lib.rs b/litellm-rust/crates/core-utils/src/lib.rs index ceb0e9eb3f2..a937f55654e 100644 --- a/litellm-rust/crates/core-utils/src/lib.rs +++ b/litellm-rust/crates/core-utils/src/lib.rs @@ -1,7 +1,9 @@ pub mod call_arguments; pub mod core_helpers; +pub mod dot_notation_indexing; pub mod exception_mapping_utils; pub mod get_llm_provider_logic; +pub mod get_provider_specific_headers; pub mod params; pub mod prompt_templates; pub mod secret_redaction; diff --git a/litellm-rust/crates/core/src/messages/common_utils.rs b/litellm-rust/crates/core/src/messages/common_utils.rs index dcefa3ebffc..015e026f6da 100644 --- a/litellm-rust/crates/core/src/messages/common_utils.rs +++ b/litellm-rust/crates/core/src/messages/common_utils.rs @@ -1,5 +1,5 @@ use litellm_http::request::string_headers as shared_string_headers; -pub(super) use litellm_http::request::{has_bearer_auth, has_header, truncate_error_body}; +pub(super) use litellm_http::request::truncate_error_body; use litellm_llms::{ anthropic::experimental_pass_through::messages::transformation::ANTHROPIC_MESSAGES_CONFIG, azure_ai::anthropic::messages_transformation::AZURE_ANTHROPIC_MESSAGES_CONFIG, diff --git a/litellm-rust/crates/core/src/messages/error.rs b/litellm-rust/crates/core/src/messages/error.rs index 51fb764032c..2a9723beb38 100644 --- a/litellm-rust/crates/core/src/messages/error.rs +++ b/litellm-rust/crates/core/src/messages/error.rs @@ -1,3 +1,5 @@ +use std::sync::Arc; + use litellm_llms::base_llm::chat::transformation::Error as LlmError; #[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)] @@ -18,8 +20,34 @@ pub enum Error { Transport(#[from] litellm_http::transport::Error), #[error(transparent)] Headers(#[from] litellm_http::request::HeaderError), + #[error(transparent)] + Secret(#[from] SecretError), } +#[derive(Clone, Debug, thiserror::Error)] +#[error(transparent)] +pub struct SecretError(Arc); + +impl SecretError { + pub fn source_error(&self) -> &litellm_secrets::Error { + &self.0 + } +} + +impl From for Error { + fn from(error: litellm_secrets::Error) -> Self { + Self::Secret(SecretError(Arc::new(error))) + } +} + +impl PartialEq for SecretError { + fn eq(&self, other: &Self) -> bool { + Arc::ptr_eq(&self.0, &other.0) + } +} + +impl Eq for SecretError {} + impl From for Error { fn from(error: LlmError) -> Self { match error { diff --git a/litellm-rust/crates/core/src/messages/mod.rs b/litellm-rust/crates/core/src/messages/mod.rs index 289f79109dd..8795d4f8507 100644 --- a/litellm-rust/crates/core/src/messages/mod.rs +++ b/litellm-rust/crates/core/src/messages/mod.rs @@ -12,6 +12,9 @@ mod common_utils; mod handler; mod prepare; pub mod route; +use std::sync::Arc; + +use litellm_secrets::source::EnvironmentSecrets; use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; use route::{LocalMessagesHost, MessagesCall, MessagesOutput, messages_machine}; use serde_json::Value; @@ -31,9 +34,12 @@ pub async fn messages(request: MessagesRequest<'_>) -> Result Ok(*message), MessagesOutput::Streamed => Err(Error::Unsupported( "streamed responses need a streaming host", diff --git a/litellm-rust/crates/core/src/messages/prepare.rs b/litellm-rust/crates/core/src/messages/prepare.rs index 850f9108869..dc4b3562e3f 100644 --- a/litellm-rust/crates/core/src/messages/prepare.rs +++ b/litellm-rust/crates/core/src/messages/prepare.rs @@ -1,51 +1,102 @@ -use litellm_core_utils::get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}; -use litellm_llms::base_llm::anthropic_messages::transformation::{ - BaseAnthropicMessagesConfig, MessagesAuthStrategy, +use litellm_core_utils::{ + dot_notation_indexing::delete_nested_value, + get_llm_provider_logic::{CustomLlmProvider, get_custom_llm_provider}, + get_provider_specific_headers::get_provider_specific_headers, + settings::Lookup, +}; +use litellm_llms::{ + anthropic::experimental_pass_through::messages::handler::shape_anthropic_messages_request, + base_llm::anthropic_messages::transformation::{ + BaseAnthropicMessagesConfig, MessagesTransformContext, + }, }; use litellm_types::llms::anthropic_messages::anthropic_request::AnthropicMessagesRequest; use serde_json::{Map, Value}; use super::{ Error, - common_utils::{has_bearer_auth, has_header, messages_provider_config, string_headers}, + common_utils::{messages_provider_config, string_headers}, }; use crate::messages::types::{MessagesRequest, ProviderMessagesRequest}; -pub(super) fn prepare_provider_request( - request: MessagesRequest<'_>, -) -> Result { - let provider_info = get_custom_llm_provider(request.model, request.custom_llm_provider) +pub(super) struct ResolvedProvider<'a> { + pub(super) model: &'a str, + pub(super) provider: &'a str, + pub(super) config: &'static dyn BaseAnthropicMessagesConfig, +} + +pub(super) fn resolve_provider<'a>( + model: &'a str, + custom_llm_provider: Option<&'a str>, +) -> Result, Error> { + let CustomLlmProvider { + model, + custom_llm_provider: provider, + } = get_custom_llm_provider(model, custom_llm_provider) .or_else(|| { - request - .custom_llm_provider - .map(|provider| CustomLlmProvider { - model: request.model, - custom_llm_provider: provider, - }) + custom_llm_provider.map(|provider| CustomLlmProvider { + model, + custom_llm_provider: provider, + }) }) .ok_or_else(|| { Error::InvalidProvider( "unable to resolve custom_llm_provider for messages request".to_string(), ) })?; - let model = provider_info.model.to_string(); - let provider = provider_info.custom_llm_provider; - let config = messages_provider_config(provider) .ok_or_else(|| Error::InvalidProvider(provider.to_string()))?; - let env_lookup = |key: &str| std::env::var(key).ok(); + Ok(ResolvedProvider { + model, + provider, + config, + }) +} - let headers = - validate_environment(config, request.extra_headers, request.api_key, &env_lookup)?; +pub(super) fn prepare_provider_request( + request: MessagesRequest<'_>, + resolved: ResolvedProvider<'_>, + secrets: &dyn Lookup, +) -> Result { + let ResolvedProvider { + model, + provider, + config, + } = resolved; + let model = model.to_string(); + let env_lookup = |key: &str| secrets.get(key); let typed_request: AnthropicMessagesRequest = - serde_json::from_value(request.body).map_err(|err| { - Error::InvalidRequest(format!("invalid Anthropic messages request: {err}")) - })?; - let transformed = config.transform_anthropic_messages_request(AnthropicMessagesRequest { - model: model.clone(), - ..typed_request - })?; + serde_json::from_value(request.body).map_err(invalid_request)?; + let sanitized = shape_anthropic_messages_request( + AnthropicMessagesRequest { + model: model.clone(), + ..typed_request + }, + request.shaping.reasoning_auto_summary, + )?; + let trimmed = + without_additional_drop_params(sanitized, &request.shaping.additional_drop_params)?; + let transformed = config.transform_anthropic_messages_request( + trimmed, + &MessagesTransformContext::new(request.shaping.capabilities, request.shaping.drop_params), + )?; + + let scoped = get_provider_specific_headers(request.provider_specific_header.as_ref(), provider); + let forwarded = string_headers(Some( + request + .extra_headers + .into_iter() + .flatten() + .chain(scoped) + .collect(), + ))?; + let authenticated = config.authenticate(forwarded, request.api_key, &env_lookup)?; + let headers = config.request_headers( + with_default_headers(authenticated, config.default_headers()), + &transformed, + ); + let body = serde_json::to_value(transformed).map_err(|err| { Error::InvalidRequest(format!( "failed to serialize Anthropic messages request: {err}" @@ -65,33 +116,371 @@ pub(super) fn prepare_provider_request( }) } -fn validate_environment( - config: &dyn BaseAnthropicMessagesConfig, - extra_headers: Option>, - api_key: Option<&str>, - env_lookup: &dyn Fn(&str) -> Option, -) -> Result, Error> { - let mut headers = string_headers(extra_headers)?; - - let auth_strategy = config.auth_strategy(); - let already_authorized = has_header(&headers, auth_strategy.header_name()) - || (config.accepts_bearer_auth() && has_bearer_auth(&headers)); - if !already_authorized { - let api_key = config.resolve_api_key(api_key, env_lookup)?; - let auth_header = match auth_strategy { - MessagesAuthStrategy::Bearer => { - ("authorization".to_string(), format!("Bearer {api_key}")) - } - MessagesAuthStrategy::Header(name) => (name.to_string(), api_key), - }; - headers.push(auth_header); - } - - for (name, value) in config.default_headers() { - if !has_header(&headers, name) { - headers.push((name.to_string(), value.to_string())); - } - } - - Ok(headers) +fn invalid_request(err: serde_json::Error) -> Error { + Error::InvalidRequest(format!("invalid Anthropic messages request: {err}")) +} + +fn without_additional_drop_params( + request: AnthropicMessagesRequest, + paths: &[String], +) -> Result { + if paths.is_empty() { + return Ok(request); + } + let Value::Object(fields) = serde_json::to_value(request).map_err(invalid_request)? else { + return Err(Error::InvalidRequest( + "Anthropic messages request did not serialize to an object".to_string(), + )); + }; + let (required, optional): (Map, Map) = fields + .into_iter() + .partition(|(key, _)| matches!(key.as_str(), "model" | "messages")); + let trimmed = paths.iter().fold(Value::Object(optional), |body, path| { + delete_nested_value(body, path) + }); + let merged: Map = required + .into_iter() + .chain(trimmed.as_object().cloned().unwrap_or_default()) + .collect(); + serde_json::from_value(Value::Object(merged)).map_err(invalid_request) +} + +fn with_default_headers( + headers: Vec<(String, String)>, + defaults: &[(&str, &str)], +) -> Vec<(String, String)> { + let missing: Vec<(String, String)> = defaults + .iter() + .filter(|(name, _)| { + !headers + .iter() + .any(|(header, _)| header.eq_ignore_ascii_case(name)) + }) + .map(|(name, value)| ((*name).to_string(), (*value).to_string())) + .collect(); + headers.into_iter().chain(missing).collect() +} + +#[cfg(test)] +mod tests { + use litellm_types::utils::ProviderSpecificHeaders; + use rstest::{fixture, rstest}; + use serde_json::json; + + use super::*; + use crate::messages::types::MessagesShaping; + + #[fixture] + fn shaping() -> MessagesShaping { + MessagesShaping::default() + } + + fn prepare(request: MessagesRequest<'_>) -> Result { + prepare_with_secrets(request, &|_: &str| None) + } + + fn prepare_with_secrets( + request: MessagesRequest<'_>, + secrets: &dyn Lookup, + ) -> Result { + let resolved = resolve_provider(request.model, request.custom_llm_provider)?; + prepare_provider_request(request, resolved, secrets) + } + + #[rstest] + #[case::api_key( + &[("ANTHROPIC_API_KEY", "sk-secret")], + &[("x-api-key", "sk-secret")], + "https://api.anthropic.com/v1/messages" + )] + #[case::auth_token( + &[("ANTHROPIC_AUTH_TOKEN", "token")], + &[("authorization", "Bearer token")], + "https://api.anthropic.com/v1/messages" + )] + #[case::api_base( + &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_API_BASE", "https://gateway.test")], + &[("x-api-key", "sk-secret")], + "https://gateway.test/v1/messages" + )] + #[case::sdk_base_url( + &[("ANTHROPIC_API_KEY", "sk-secret"), ("ANTHROPIC_BASE_URL", "https://sdk.test")], + &[("x-api-key", "sk-secret")], + "https://sdk.test/v1/messages" + )] + fn credentials_and_base_come_from_the_resolved_secrets( + shaping: MessagesShaping, + #[case] secrets: &[(&str, &str)], + #[case] expected_auth: &[(&str, &str)], + #[case] expected_url: &str, + ) { + let lookup = |name: &str| { + secrets + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + }; + let prepared = prepare_with_secrets( + MessagesRequest { + model: "claude-test", + body: json!({"model": "claude-test", "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + api_key: None, + api_base: None, + custom_llm_provider: Some("anthropic"), + extra_headers: None, + provider_specific_header: None, + timeout: None, + shaping, + }, + &lookup, + ) + .unwrap(); + let auth: Vec<(&str, &str)> = prepared + .upstream_headers + .iter() + .filter(|(name, _)| matches!(name.as_str(), "x-api-key" | "authorization")) + .map(|(name, value)| (name.as_str(), value.as_str())) + .collect(); + assert_eq!( + (auth.as_slice(), prepared.url.as_str()), + (expected_auth, expected_url) + ); + } + + fn prepared_body(body: Value, shaping: MessagesShaping) -> Result { + prepare(MessagesRequest { + model: "anthropic/claude-test", + body, + api_key: Some("sk-test"), + api_base: Some("https://anthropic.test"), + custom_llm_provider: Some("anthropic"), + extra_headers: None, + provider_specific_header: None, + timeout: None, + shaping, + }) + .map(|prepared| prepared.body) + } + + #[rstest] + #[case::nothing_forwarded( + &[], + &[("x-version", "1"), ("content-type", "application/json")], + &[("x-version", "1"), ("content-type", "application/json")], + )] + #[case::forwarded_header_wins_in_any_case( + &[("X-Version", "custom"), ("x-api-key", "k")], + &[("x-version", "1"), ("content-type", "application/json")], + &[("X-Version", "custom"), ("x-api-key", "k"), ("content-type", "application/json")], + )] + #[case::no_defaults(&[("x-api-key", "k")], &[], &[("x-api-key", "k")])] + fn default_headers_fill_only_missing_names( + #[case] forwarded: &[(&str, &str)], + #[case] defaults: &[(&str, &str)], + #[case] expected: &[(&str, &str)], + ) { + let owned = |headers: &[(&str, &str)]| -> Vec<(String, String)> { + headers + .iter() + .map(|(name, value)| ((*name).to_string(), (*value).to_string())) + .collect() + }; + assert_eq!( + with_default_headers(owned(forwarded), defaults), + owned(expected) + ); + } + + #[rstest] + #[case::top_level_and_nested_paths( + json!({ + "max_tokens": 1024, + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "context_management": {"edits": [{"type": "clear_thinking_20251015"}]}, + "metadata": {"user_id": "u1"}, + "tools": [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}] + }), + &["thinking", "context_management", "tools[*].input_examples"], + json!({ + "max_tokens": 1024, + "metadata": {"user_id": "u1"}, + "tools": [{"name": "lookup", "input_schema": {"type": "object"}}] + }), + )] + #[case::no_paths( + json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}), + &[], + json!({"max_tokens": 16, "safeguards": [{"type": "dangerous_tool_use"}]}), + )] + #[case::model_and_messages_are_never_dropped( + json!({"max_tokens": 16}), + &["model", "messages", "messages[0].content"], + json!({"max_tokens": 16}), + )] + fn prepared_body_drops_configured_paths( + shaping: MessagesShaping, + #[case] fields: Value, + #[case] additional_drop_params: &[&str], + #[case] expected_fields: Value, + ) { + let with_messages = |fields: Value| -> Value { + let Value::Object(fields) = fields else { + unreachable!() + }; + Value::Object( + [ + ("model".to_string(), json!("claude-test")), + ( + "messages".to_string(), + json!([{"role": "user", "content": "hi"}]), + ), + ] + .into_iter() + .chain(fields) + .collect(), + ) + }; + let shaping = MessagesShaping { + additional_drop_params: additional_drop_params + .iter() + .map(ToString::to_string) + .collect(), + ..shaping + }; + assert_eq!( + prepared_body(with_messages(fields), shaping), + Ok(with_messages(expected_fields)) + ); + } + + #[rstest] + #[case::model_prefix_picks_the_provider( + "azure_ai/claude-test", + None, + &[("x-priority", "extra"), ("x-scoped", "azure_ai")] + )] + #[case::explicit_provider( + "claude-test", + Some("anthropic"), + &[("x-priority", "scoped"), ("x-scoped", "anthropic")] + )] + #[case::provider_prefix_on_an_anthropic_model( + "anthropic/claude-test", + None, + &[("x-priority", "scoped"), ("x-scoped", "anthropic")] + )] + fn provider_specific_headers_follow_the_resolved_provider( + shaping: MessagesShaping, + #[case] model: &str, + #[case] custom_llm_provider: Option<&str>, + #[case] expected: &[(&str, &str)], + ) { + let configured: ProviderSpecificHeaders = serde_json::from_value(json!([ + {"custom_llm_provider": "azure_ai", "extra_headers": {"x-scoped": "azure_ai"}}, + {"custom_llm_provider": "anthropic", "extra_headers": {"x-scoped": "anthropic", "x-priority": "scoped"}} + ])) + .unwrap(); + let prepared = prepare(MessagesRequest { + model, + body: json!({"model": model, "messages": [{"role": "user", "content": "hi"}], "max_tokens": 16}), + api_key: Some("sk-test"), + api_base: Some("https://resource.services.ai.azure.com"), + custom_llm_provider, + extra_headers: Some(serde_json::from_value(json!({"x-priority": "extra"})).unwrap()), + provider_specific_header: Some(configured), + timeout: None, + shaping, + }) + .unwrap(); + let caller_headers: Vec<(&str, &str)> = prepared + .upstream_headers + .iter() + .filter(|(name, _)| matches!(name.as_str(), "x-priority" | "x-scoped")) + .map(|(name, value)| (name.as_str(), value.as_str())) + .collect(); + assert_eq!(caller_headers, expected); + } + + #[rstest] + fn prepared_body_carries_the_provider_stripped_model(shaping: MessagesShaping) { + assert_eq!( + prepared_body( + json!({ + "model": "anthropic/claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16 + }), + shaping, + ), + Ok(json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16 + })) + ); + } + + #[rstest] + fn dropped_thinking_display_is_not_restored_by_auto_summary(shaping: MessagesShaping) { + let shaping = MessagesShaping { + reasoning_auto_summary: true, + additional_drop_params: vec!["thinking.display".to_string()], + ..shaping + }; + assert_eq!( + prepared_body( + json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 4096, + "thinking": {"type": "enabled", "budget_tokens": 2048} + }), + shaping, + ), + Ok(json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 4096, + "thinking": {"type": "enabled", "budget_tokens": 2048} + })) + ); + } + + #[rstest] + fn dropping_an_invalid_metadata_user_id_does_not_skip_its_validation(shaping: MessagesShaping) { + let shaping = MessagesShaping { + additional_drop_params: vec!["metadata.user_id".to_string()], + ..shaping + }; + assert!(matches!( + prepared_body( + json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16, + "metadata": {"user_id": 123} + }), + shaping, + ), + Err(Error::InvalidRequest(_)) + )); + } + + #[rstest] + fn prepared_body_rejects_invalid_metadata_before_the_call(shaping: MessagesShaping) { + assert_eq!( + prepared_body( + json!({ + "model": "claude-test", + "messages": [{"role": "user", "content": "hi"}], + "max_tokens": 16, + "metadata": {"user_id": 123} + }), + shaping, + ), + Err(Error::InvalidRequest( + "metadata.user_id must be a string, got 123".to_string() + )) + ); + } } diff --git a/litellm-rust/crates/core/src/messages/route.rs b/litellm-rust/crates/core/src/messages/route.rs index 838b56fcb4b..8cd3eaf3aa3 100644 --- a/litellm-rust/crates/core/src/messages/route.rs +++ b/litellm-rust/crates/core/src/messages/route.rs @@ -1,4 +1,7 @@ -use std::{sync::Mutex, time::Duration}; +use std::{ + sync::{Arc, Mutex}, + time::Duration, +}; use bytes::Bytes; use litellm_auth::SecretValue; @@ -9,15 +12,19 @@ use litellm_host::{ machine::{HostChannel, MachineFault, RouteMachine}, route::Route, }; -use litellm_types::llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse; +use litellm_secrets::source::SecretSource; +use litellm_types::{ + llms::anthropic_messages::anthropic_response::AnthropicMessagesResponse, + utils::ProviderSpecificHeaders, +}; use serde_json::{Map, Value}; use super::{ Error, common_utils::messages_provider_config, handler::{decode_response, network, provider_error, send}, - prepare::prepare_provider_request, - types::MessagesRequest, + prepare::{prepare_provider_request, resolve_provider}, + types::{MessagesRequest, MessagesShaping}, }; use crate::constants::ANTHROPIC_MESSAGES_PROVIDER; @@ -38,7 +45,9 @@ pub struct MessagesCall { pub api_base: Option, pub custom_llm_provider: Option, pub extra_headers: Option>, + pub provider_specific_header: Option, pub timeout: Option, + pub shaping: MessagesShaping, } impl MessagesCall { @@ -120,22 +129,33 @@ impl Host for LocalMessagesHost { } } -pub fn messages_machine() -> MessagesMachine { - RouteMachine::new(|host| Box::pin(execute(host))) +pub fn messages_machine(secrets: Arc) -> MessagesMachine { + RouteMachine::new(move |host| Box::pin(execute(host, secrets.clone()))) } -async fn execute(host: MessagesHost) -> Result { +async fn execute( + host: MessagesHost, + secrets: Arc, +) -> Result { let MessagesOpResult::Request(call) = host.route(MessagesOp::ProjectRequest).await?; let stream = call.streams(); - let request = prepare_provider_request(MessagesRequest { - model: &call.model, - body: Value::Object(call.body.clone()), - api_key: call.api_key.as_deref(), - api_base: call.api_base.as_deref(), - custom_llm_provider: call.custom_llm_provider.as_deref(), - extra_headers: call.extra_headers.clone(), - timeout: call.timeout, - })?; + let resolved = resolve_provider(&call.model, call.custom_llm_provider.as_deref())?; + let secrets = secrets.resolve(resolved.config.secret_names()).await?; + let request = prepare_provider_request( + MessagesRequest { + model: &call.model, + body: Value::Object(call.body.clone()), + api_key: call.api_key.as_deref(), + api_base: call.api_base.as_deref(), + custom_llm_provider: call.custom_llm_provider.as_deref(), + extra_headers: call.extra_headers.clone(), + provider_specific_header: call.provider_specific_header.clone(), + timeout: call.timeout, + shaping: call.shaping.clone(), + }, + resolved, + secrets.as_ref(), + )?; if stream && request.provider != ANTHROPIC_MESSAGES_PROVIDER { return Err(Error::Unsupported("streaming messages for this provider")); } diff --git a/litellm-rust/crates/core/src/messages/tests.rs b/litellm-rust/crates/core/src/messages/tests.rs index 057b42a316c..ce48752864a 100644 --- a/litellm-rust/crates/core/src/messages/tests.rs +++ b/litellm-rust/crates/core/src/messages/tests.rs @@ -1,5 +1,8 @@ -use std::time::Duration; +use std::{sync::Arc, time::Duration}; +use futures_util::future::BoxFuture; +use litellm_http::request::{has_bearer_auth, has_header}; +use litellm_secrets::{SecretValue, source::SecretSource}; use serde_json::{Map, Value, json}; use tokio::{ io::{AsyncReadExt, AsyncWriteExt}, @@ -8,12 +11,132 @@ use tokio::{ use super::{ Error, - common_utils::{ - has_bearer_auth, has_header, messages_provider_config, string_headers, truncate_error_body, - }, + common_utils::{messages_provider_config, string_headers, truncate_error_body}, messages, + route::{LocalMessagesHost, MessagesCall, MessagesOutput, messages_machine}, }; -use crate::messages::types::MessagesRequest; +use crate::messages::types::{MessagesRequest, MessagesShaping}; + +struct RecordingSecrets { + values: Vec<(&'static str, String)>, + fails: bool, + requested: std::sync::Mutex>, +} + +impl RecordingSecrets { + fn new(values: Vec<(&'static str, String)>, fails: bool) -> Self { + Self { + values, + fails, + requested: std::sync::Mutex::new(Vec::new()), + } + } +} + +impl SecretSource for RecordingSecrets { + fn get_secret_str<'a>( + &'a self, + name: &'a str, + ) -> BoxFuture<'a, Result, litellm_secrets::Error>> { + Box::pin(async move { + self.requested.lock().unwrap().push(name.to_string()); + if self.fails { + return Err(litellm_secrets::Error::ManagedSecretMissing); + } + Ok(self + .values + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| SecretValue::new(value.clone()))) + }) + } +} + +fn secrets_call() -> MessagesCall { + let Value::Object(body) = json!({ + "model": "claude-sonnet-4-5", + "max_tokens": 16, + "messages": [{"role": "user", "content": "hi"}] + }) else { + unreachable!("literal object") + }; + MessagesCall { + model: "claude-sonnet-4-5".into(), + body, + api_key: None, + api_base: None, + custom_llm_provider: Some("anthropic".into()), + extra_headers: None, + provider_specific_header: None, + timeout: Some(Duration::from_secs(5)), + shaping: MessagesShaping::default(), + } +} + +#[tokio::test] +async fn route_reads_the_provider_credential_and_base_from_the_secret_source() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("binds"); + let addr = listener.local_addr().expect("addr"); + let server = tokio::spawn(async move { + let (mut socket, _) = listener.accept().await.expect("accepts request"); + let request = read_http_request(&mut socket).await; + let response_body = r#"{"id":"msg_1","type":"message","role":"assistant","content":[],"model":"claude-sonnet-4-5","stop_reason":"end_turn","usage":{"input_tokens":1,"output_tokens":1}}"#; + socket + .write_all(write_response(response_body).as_bytes()) + .await + .expect("writes response"); + request + }); + let secrets = Arc::new(RecordingSecrets::new( + vec![ + ("ANTHROPIC_API_KEY", "sk-from-manager".to_string()), + ("ANTHROPIC_BASE_URL", format!("http://{addr}")), + ], + false, + )); + + let output = litellm_host::run::run( + messages_machine(secrets.clone()), + &LocalMessagesHost::new(secrets_call()), + ) + .await + .expect("messages request succeeds"); + + assert!(matches!(output, MessagesOutput::Message(_))); + let request = server.await.expect("server task completes"); + assert!( + request + .to_ascii_lowercase() + .contains("x-api-key: sk-from-manager"), + "{request}" + ); + let requested = secrets.requested.lock().unwrap().clone(); + assert_eq!( + requested, + messages_provider_config("anthropic") + .unwrap() + .secret_names() + .iter() + .map(ToString::to_string) + .collect::>() + ); +} + +#[tokio::test] +async fn route_surfaces_a_secret_manager_failure_before_the_call() { + let Err(error) = litellm_host::run::run( + messages_machine(Arc::new(RecordingSecrets::new(Vec::new(), true))), + &LocalMessagesHost::new(secrets_call()), + ) + .await + else { + panic!("a secret manager failure fails the call"); + }; + assert!( + matches!(&error, Error::Secret(source) if matches!(source.source_error(), litellm_secrets::Error::ManagedSecretMissing)), + "{error:?}" + ); +} async fn read_http_request(socket: &mut TcpStream) -> String { let mut request = Vec::new(); @@ -159,7 +282,9 @@ async fn messages_round_trip_builds_azure_request_and_passes_response_through() api_base: Some(&format!("http://{addr}")), custom_llm_provider: Some("azure_ai"), extra_headers: None, + provider_specific_header: None, timeout: Some(Duration::from_secs(5)), + shaping: MessagesShaping::default(), }) .await .expect("messages request succeeds"); @@ -215,7 +340,9 @@ async fn messages_round_trip_builds_native_anthropic_request() { api_base: Some(&format!("http://{addr}")), custom_llm_provider: Some("anthropic"), extra_headers: None, + provider_specific_header: None, timeout: Some(Duration::from_secs(5)), + shaping: MessagesShaping::default(), }) .await .expect("messages request succeeds"); @@ -268,7 +395,9 @@ async fn messages_does_not_duplicate_auth_when_x_api_key_supplied() { api_base: Some(&format!("http://{addr}")), custom_llm_provider: Some("azure_ai"), extra_headers: Some(headers), + provider_specific_header: None, timeout: Some(Duration::from_secs(5)), + shaping: MessagesShaping::default(), }) .await .expect("messages request succeeds"); @@ -322,7 +451,9 @@ async fn messages_forwards_entra_id_bearer_without_requiring_api_key() { api_base: Some(&format!("http://{addr}")), custom_llm_provider: Some("azure_ai"), extra_headers: Some(headers), + provider_specific_header: None, timeout: Some(Duration::from_secs(5)), + shaping: MessagesShaping::default(), }) .await .expect("entra id request succeeds without api key"); @@ -346,7 +477,9 @@ async fn messages_requires_auth_when_no_key_and_no_header() { api_base: Some("http://127.0.0.1:1"), custom_llm_provider: Some("azure_ai"), extra_headers: None, + provider_specific_header: None, timeout: Some(Duration::from_millis(50)), + shaping: MessagesShaping::default(), }) .await .expect_err("missing auth errors"); @@ -384,7 +517,9 @@ async fn messages_ignores_malformed_authorization_and_uses_api_key() { api_base: Some(&format!("http://{addr}")), custom_llm_provider: Some("azure_ai"), extra_headers: Some(headers), + provider_specific_header: None, timeout: Some(Duration::from_secs(5)), + shaping: MessagesShaping::default(), }) .await .expect("falls back to api key"); @@ -425,7 +560,9 @@ async fn messages_maps_provider_error_status_to_http_error() { api_base: Some(&format!("http://{addr}")), custom_llm_provider: Some("azure_ai"), extra_headers: None, + provider_specific_header: None, timeout: Some(Duration::from_secs(5)), + shaping: MessagesShaping::default(), }) .await .expect_err("provider error propagates"); @@ -445,7 +582,9 @@ async fn messages_rejects_unsupported_provider() { api_base: Some("http://127.0.0.1:1"), custom_llm_provider: Some("openai"), extra_headers: None, + provider_specific_header: None, timeout: Some(Duration::from_millis(50)), + shaping: MessagesShaping::default(), }) .await .expect_err("unsupported provider errors"); diff --git a/litellm-rust/crates/core/src/messages/types.rs b/litellm-rust/crates/core/src/messages/types.rs index a73ceffad7a..4a5dd2926e0 100644 --- a/litellm-rust/crates/core/src/messages/types.rs +++ b/litellm-rust/crates/core/src/messages/types.rs @@ -1,8 +1,25 @@ use std::time::Duration; -use litellm_llms::base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig; +use litellm_llms::{ + anthropic::common_utils::AnthropicModelCapabilities, + base_llm::anthropic_messages::transformation::BaseAnthropicMessagesConfig, +}; +use litellm_types::utils::ProviderSpecificHeaders; +use serde::{Deserialize, Serialize}; use serde_json::{Map, Value}; +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct MessagesShaping { + #[serde(default)] + pub capabilities: AnthropicModelCapabilities, + #[serde(default)] + pub drop_params: bool, + #[serde(default)] + pub reasoning_auto_summary: bool, + #[serde(default)] + pub additional_drop_params: Vec, +} + pub struct MessagesRequest<'a> { pub model: &'a str, pub body: Value, @@ -10,7 +27,9 @@ pub struct MessagesRequest<'a> { pub api_base: Option<&'a str>, pub custom_llm_provider: Option<&'a str>, pub extra_headers: Option>, + pub provider_specific_header: Option, pub timeout: Option, + pub shaping: MessagesShaping, } pub struct ProviderMessagesRequest { @@ -22,3 +41,86 @@ pub struct ProviderMessagesRequest { pub upstream_headers: Vec<(String, String)>, pub timeout: Option, } + +#[cfg(test)] +mod tests { + use litellm_llms::anthropic::common_utils::SupportedEffortTiers; + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::nothing_projected(json!({}), MessagesShaping::default())] + #[case::only_drop_params( + json!({"drop_params": true}), + MessagesShaping { drop_params: true, ..MessagesShaping::default() }, + )] + #[case::only_reasoning_auto_summary( + json!({"reasoning_auto_summary": true}), + MessagesShaping { reasoning_auto_summary: true, ..MessagesShaping::default() }, + )] + #[case::only_additional_drop_params( + json!({"additional_drop_params": ["tools[*].input_examples"]}), + MessagesShaping { + additional_drop_params: vec!["tools[*].input_examples".to_string()], + ..MessagesShaping::default() + }, + )] + #[case::partial_capabilities( + json!({"capabilities": {"supports_reasoning": true}}), + MessagesShaping { + capabilities: AnthropicModelCapabilities { + supports_reasoning: true, + ..AnthropicModelCapabilities::default() + }, + ..MessagesShaping::default() + }, + )] + #[case::everything_the_python_host_projects( + json!({ + "capabilities": { + "supports_reasoning": true, + "supports_adaptive_thinking": true, + "thinking_always_on": false, + "supports_legacy_thinking": false, + "supports_output_config": true, + "supports_sampling_params": false, + "supports_speed": true, + "effort_tiers": {"minimal": false, "low": true, "medium": true, "high": true, "xhigh": true, "max": false} + }, + "drop_params": true, + "reasoning_auto_summary": true, + "additional_drop_params": ["metadata.user_id", "thinking"] + }), + MessagesShaping { + capabilities: AnthropicModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + thinking_always_on: false, + supports_legacy_thinking: false, + supports_output_config: true, + supports_sampling_params: false, + supports_speed: true, + effort_tiers: SupportedEffortTiers { + minimal: false, + low: true, + medium: true, + high: true, + xhigh: true, + max: false, + }, + }, + drop_params: true, + reasoning_auto_summary: true, + additional_drop_params: vec!["metadata.user_id".to_string(), "thinking".to_string()], + }, + )] + fn shaping_deserializes_with_defaults_for_absent_fields( + #[case] projected: Value, + #[case] expected: MessagesShaping, + ) { + let shaping: MessagesShaping = serde_json::from_value(projected).unwrap(); + assert_eq!(shaping, expected); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/common_utils.rs b/litellm-rust/crates/llms/src/anthropic/common_utils.rs new file mode 100644 index 00000000000..a2234e0df03 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/common_utils.rs @@ -0,0 +1,1568 @@ +use litellm_types::llms::anthropic_messages::anthropic_request::{ + AnthropicMessage, ContentBlock, MessageContent, +}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::anthropic::ANTHROPIC_OAUTH_TOKEN_PREFIX; + +pub const ANTHROPIC_OAUTH_BETA_HEADER: &str = "oauth-2025-04-20"; +pub const ANTHROPIC_ADVISOR_TOOL_TYPE: &str = "advisor_20260301"; +pub const ANTHROPIC_TOOL_SEARCH_TOOL_TYPES: [&str; 2] = [ + "tool_search_tool_regex_20251119", + "tool_search_tool_bm25_20251119", +]; +pub const ENCRYPTED_REASONING_SIGNATURE_PREFIX: &str = "litellm_encrypted_reasoning:"; +const THOUGHT_SIGNATURE_SEPARATOR: &str = "__thought__"; + +pub mod beta { + pub const CONTEXT_MANAGEMENT_2025_06_27: &str = "context-management-2025-06-27"; + pub const COMPACT_2026_01_12: &str = "compact-2026-01-12"; + pub const COMPACT_2026_09_04: &str = "compact-2026-09-04"; + pub const STRUCTURED_OUTPUT: &str = "structured-outputs-2025-11-13"; + pub const ADVANCED_TOOL_USE_2025_11_20: &str = "advanced-tool-use-2025-11-20"; + pub const FAST_MODE_2026_02_01: &str = "fast-mode-2026-02-01"; + pub const ADVISOR_TOOL_2026_03_01: &str = "advisor-tool-2026-03-01"; + pub const PER_TURN_CONTROL_2026_07_01: &str = "per-turn-control-2026-07-01"; +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] +pub enum EffortLevel { + Low, + Medium, + High, + Xhigh, + Max, +} + +impl EffortLevel { + pub fn as_str(self) -> &'static str { + match self { + Self::Low => "low", + Self::Medium => "medium", + Self::High => "high", + Self::Xhigh => "xhigh", + Self::Max => "max", + } + } + + pub fn parse(value: &str) -> Option { + match value { + "low" => Some(Self::Low), + "medium" => Some(Self::Medium), + "high" => Some(Self::High), + "xhigh" => Some(Self::Xhigh), + "max" => Some(Self::Max), + _ => None, + } + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct SupportedEffortTiers { + #[serde(default)] + pub minimal: bool, + #[serde(default)] + pub low: bool, + #[serde(default)] + pub medium: bool, + #[serde(default)] + pub high: bool, + #[serde(default)] + pub xhigh: bool, + #[serde(default)] + pub max: bool, +} + +impl SupportedEffortTiers { + pub fn any(self) -> bool { + self.minimal || self.low || self.medium || self.high || self.xhigh || self.max + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct AnthropicModelCapabilities { + #[serde(default)] + pub supports_reasoning: bool, + #[serde(default)] + pub supports_adaptive_thinking: bool, + #[serde(default)] + pub thinking_always_on: bool, + #[serde(default)] + pub supports_legacy_thinking: bool, + #[serde(default)] + pub supports_output_config: bool, + #[serde(default = "default_true")] + pub supports_sampling_params: bool, + #[serde(default)] + pub supports_speed: bool, + #[serde(default)] + pub effort_tiers: SupportedEffortTiers, +} + +fn default_true() -> bool { + true +} + +impl Default for AnthropicModelCapabilities { + fn default() -> Self { + Self { + supports_reasoning: false, + supports_adaptive_thinking: false, + thinking_always_on: false, + supports_legacy_thinking: false, + supports_output_config: false, + supports_sampling_params: true, + supports_speed: false, + effort_tiers: SupportedEffortTiers::default(), + } + } +} + +impl AnthropicModelCapabilities { + pub fn supports_effort_tier(&self, level: EffortLevel) -> bool { + match level { + EffortLevel::Low => self.effort_tiers.low, + EffortLevel::Medium => self.effort_tiers.medium, + EffortLevel::High => self.effort_tiers.high, + EffortLevel::Xhigh => self.effort_tiers.xhigh, + EffortLevel::Max => self.effort_tiers.max, + } + } + + pub fn supports_effort_param(&self) -> bool { + self.supports_output_config || self.effort_tiers.any() + } + + pub fn effort_level_rejection(&self, effort: &str, model: &str) -> Option { + match effort { + "max" if !(self.supports_adaptive_thinking || self.effort_tiers.max) => Some(format!( + "effort='max' is not supported by this model. Got model: {model}" + )), + "xhigh" if !self.effort_tiers.xhigh => Some(format!( + "effort='xhigh' is not supported by this model. Got model: {model}" + )), + _ => None, + } + } +} + +pub fn is_anthropic_oauth_key(value: &str) -> bool { + value + .strip_prefix("Bearer ") + .unwrap_or(value) + .starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX) +} + +pub fn split_beta_values(header: Option<&str>) -> impl Iterator + '_ { + header + .into_iter() + .flat_map(|value| value.split(',')) + .map(str::trim) + .filter(|piece| !piece.is_empty()) + .map(str::to_string) +} + +pub fn join_beta_values(values: impl IntoIterator) -> String { + let mut values: Vec = values.into_iter().collect(); + values.sort(); + values.dedup(); + values.join(",") +} + +pub fn is_tool_search_used(tools: Option<&[Value]>) -> bool { + tools.into_iter().flatten().any(|tool| { + tool.get("type") + .and_then(Value::as_str) + .is_some_and(|tool_type| ANTHROPIC_TOOL_SEARCH_TOOL_TYPES.contains(&tool_type)) + }) +} + +pub fn has_advisor_tool(tools: Option<&[Value]>) -> bool { + tools + .into_iter() + .flatten() + .any(|tool| tool.get("type").and_then(Value::as_str) == Some(ANTHROPIC_ADVISOR_TOOL_TYPE)) +} + +pub fn requires_native_compaction_beta( + compaction: Option<&Value>, + messages: &[AnthropicMessage], +) -> bool { + compaction.is_some() + || messages + .iter() + .flat_map(AnthropicMessage::blocks) + .any(|block| { + block.is_type("compaction") + && block.signature.as_deref().is_some_and(|s| !s.is_empty()) + }) +} + +fn is_blank(text: Option<&str>) -> bool { + text.is_none_or(|text| text.trim().is_empty()) +} + +fn is_empty_text_block(block: &ContentBlock) -> bool { + block.is_type("text") && is_blank(block.text.as_deref()) +} + +pub fn is_empty_thinking_block(block: &ContentBlock) -> bool { + block.is_type("thinking") && is_blank(block.thinking.as_deref()) +} + +fn retain_blocks( + messages: Vec, + keep: impl Fn(&ContentBlock) -> bool, +) -> Vec { + messages + .into_iter() + .filter_map(|message| match message.content { + MessageContent::Text(_) => Some(message), + MessageContent::Blocks(ref blocks) => { + let kept: Vec = + blocks.iter().filter(|block| keep(block)).cloned().collect(); + if kept.len() == blocks.len() { + return Some(message); + } + (!kept.is_empty()).then(|| message.with_blocks(kept)) + } + }) + .collect() +} + +pub fn strip_empty_content_blocks(messages: Vec) -> Vec { + retain_blocks(messages, |block| { + !is_empty_text_block(block) && !is_empty_thinking_block(block) + }) +} + +pub fn normalize_anthropic_tool_use_id(raw_id: &str) -> String { + let base = raw_id + .split_once(THOUGHT_SIGNATURE_SEPARATOR) + .map_or(raw_id, |(base, _)| base); + let sanitized: String = base + .chars() + .map(|character| { + if character.is_ascii_alphanumeric() || matches!(character, '_' | '-') { + character + } else { + '_' + } + }) + .collect(); + if sanitized.is_empty() { + "tool_use_id".to_string() + } else { + sanitized + } +} + +fn normalized_if_changed(raw_id: Option<&str>) -> Option { + let raw_id = raw_id?; + let normalized = normalize_anthropic_tool_use_id(raw_id); + (normalized != raw_id).then_some(normalized) +} + +fn sanitize_tool_use_id_block(block: ContentBlock) -> ContentBlock { + match block.block_type.as_deref() { + Some("tool_use" | "server_tool_use") => match normalized_if_changed(block.id.as_deref()) { + Some(id) => ContentBlock { + id: Some(id), + ..block + }, + None => block, + }, + Some("tool_result") => match normalized_if_changed(block.tool_use_id.as_deref()) { + Some(tool_use_id) => ContentBlock { + tool_use_id: Some(tool_use_id), + ..block + }, + None => block, + }, + _ => block, + } +} + +fn map_blocks( + messages: Vec, + rewrite: impl Fn(Vec) -> Vec, +) -> Vec { + messages + .into_iter() + .map(|message| match message.content { + MessageContent::Blocks(blocks) => AnthropicMessage { + content: MessageContent::Blocks(rewrite(blocks)), + ..message + }, + MessageContent::Text(_) => message, + }) + .collect() +} + +pub fn sanitize_tool_use_ids(messages: Vec) -> Vec { + map_blocks(messages, |blocks| { + blocks.into_iter().map(sanitize_tool_use_id_block).collect() + }) +} + +pub fn strip_provider_specific_fields(messages: Vec) -> Vec { + map_blocks(messages, |blocks| { + blocks + .into_iter() + .map(|block| ContentBlock { + provider_specific_fields: None, + ..block + }) + .collect() + }) +} + +pub fn is_encrypted_reasoning_block(block: &ContentBlock) -> bool { + let field = match block.block_type.as_deref() { + Some("thinking") => block.signature.as_deref(), + Some("redacted_thinking") => block.data.as_deref(), + _ => None, + }; + field.is_some_and(|value| value.starts_with(ENCRYPTED_REASONING_SIGNATURE_PREFIX)) +} + +pub fn strip_encrypted_reasoning_blocks(messages: Vec) -> Vec { + retain_blocks(messages, |block| !is_encrypted_reasoning_block(block)) +} + +fn is_advisor_use(block: &ContentBlock) -> bool { + block.is_type("server_tool_use") + && block.name.as_deref() == Some("advisor") + && block.id.as_deref().is_some_and(|id| !id.is_empty()) +} + +pub fn strip_advisor_blocks(messages: Vec) -> Vec { + messages + .into_iter() + .map(|message| { + if message.role != "assistant" { + return message; + } + let MessageContent::Blocks(blocks) = &message.content else { + return message; + }; + let advisor_ids: Vec<&str> = blocks + .iter() + .filter(|block| is_advisor_use(block)) + .filter_map(|block| block.id.as_deref()) + .collect(); + if advisor_ids.is_empty() { + return message; + } + let kept: Vec = blocks + .iter() + .filter(|block| { + let is_result = block.is_type("advisor_tool_result") + && block + .tool_use_id + .as_deref() + .is_some_and(|id| advisor_ids.contains(&id)); + !is_advisor_use(block) && !is_result + }) + .cloned() + .collect(); + message.with_blocks(kept) + }) + .collect() +} + +#[derive(Deserialize)] +struct ReplayedWebSearchResult { + #[serde(default)] + url: String, + #[serde(default)] + title: String, + #[serde(default)] + snippet: String, + #[serde(default)] + encrypted_content: String, +} + +#[derive(Deserialize)] +#[serde(tag = "type")] +enum ReplayedWebSearchContent { + #[serde(rename = "web_search_tool_result_error")] + Error { + #[serde(default)] + error_code: String, + }, +} + +enum WebSearchResults { + Results(Vec), + Error(String), +} + +fn flattenable_web_search_results(block: &ContentBlock) -> Option<(&str, WebSearchResults)> { + if !block.is_type("web_search_tool_result") { + return None; + } + let tool_use_id = block.tool_use_id.as_deref()?; + let results = match block.content.as_ref()? { + Value::Array(items) => { + let results = items + .iter() + .map(|item| { + (item.get("type").and_then(Value::as_str) == Some("web_search_result")) + .then(|| { + serde_json::from_value::(item.clone()).ok() + }) + .flatten() + }) + .collect::>>()?; + if results + .iter() + .any(|result| !result.encrypted_content.is_empty()) + { + return None; + } + WebSearchResults::Results(results) + } + error @ Value::Object(_) => match serde_json::from_value(error.clone()).ok()? { + ReplayedWebSearchContent::Error { error_code } => WebSearchResults::Error(error_code), + }, + _ => return None, + }; + Some((tool_use_id, results)) +} + +fn render_web_search_results(query: &str, results: &WebSearchResults) -> String { + let header = if query.is_empty() { + "Web search results:".to_string() + } else { + format!("Web search results for '{query}':") + }; + match results { + WebSearchResults::Error(code) => { + let code = if code.is_empty() { "unavailable" } else { code }; + format!("{header}\n\nSearch failed: {code}") + } + WebSearchResults::Results(results) if results.is_empty() => { + format!("{header}\n\nNo results were returned.") + } + WebSearchResults::Results(results) => { + let body = results + .iter() + .map(|result| { + [ + (!result.title.is_empty()).then(|| format!("Title: {}", result.title)), + (!result.url.is_empty()).then(|| format!("URL: {}", result.url)), + (!result.snippet.is_empty()) + .then(|| format!("Snippet: {}", result.snippet)), + ] + .into_iter() + .flatten() + .collect::>() + .join("\n") + }) + .collect::>() + .join("\n\n"); + if body.is_empty() { + header + } else { + format!("{header}\n\n{body}") + } + } + } +} + +fn server_tool_use_query(block: &ContentBlock) -> Option<(&str, &str)> { + if !block.is_type("server_tool_use") { + return None; + } + let id = block.id.as_deref()?; + let query = match block.input.as_ref() { + None => "", + Some(Value::Object(input)) => match input.get("query") { + None => "", + Some(query) => query.as_str()?, + }, + Some(_) => return None, + }; + Some((id, query)) +} + +fn flatten_web_search_results_in_blocks(blocks: Vec) -> Vec { + let flattenable_ids: Vec<&str> = blocks + .iter() + .filter_map(flattenable_web_search_results) + .map(|(tool_use_id, _)| tool_use_id) + .collect(); + if flattenable_ids.is_empty() { + return blocks; + } + let queries: Vec<(&str, &str)> = blocks.iter().filter_map(server_tool_use_query).collect(); + blocks + .iter() + .filter_map(|block| { + if let Some((tool_use_id, results)) = flattenable_web_search_results(block) { + let query = queries + .iter() + .rfind(|(id, _)| *id == tool_use_id) + .map_or("", |(_, query)| query); + return Some(ContentBlock::text(render_web_search_results( + query, &results, + ))); + } + if let Some((id, _)) = server_tool_use_query(block) + && flattenable_ids.contains(&id) + { + return None; + } + Some(block.clone()) + }) + .collect() +} + +pub fn flatten_unencrypted_web_search_results( + messages: Vec, +) -> Vec { + map_blocks(messages, flatten_web_search_results_in_blocks) +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + use serde_json::json; + + use super::*; + + const ALL_LEVELS: [EffortLevel; 5] = [ + EffortLevel::Low, + EffortLevel::Medium, + EffortLevel::High, + EffortLevel::Xhigh, + EffortLevel::Max, + ]; + + fn apply( + sanitizer: fn(Vec) -> Vec, + messages: Value, + ) -> Value { + let parsed: Vec = serde_json::from_value(messages).unwrap(); + serde_json::to_value(sanitizer(parsed)).unwrap() + } + + fn block(value: Value) -> ContentBlock { + serde_json::from_value(value).unwrap() + } + + fn history(messages: Value) -> Vec { + serde_json::from_value(messages).unwrap() + } + + fn tools(value: Option) -> Option> { + value.map(|tools| tools.as_array().unwrap().clone()) + } + + fn tagged(encrypted: &str) -> String { + format!("{ENCRYPTED_REASONING_SIGNATURE_PREFIX}{encrypted}") + } + + fn tiers( + minimal: bool, + low: bool, + medium: bool, + high: bool, + xhigh: bool, + max: bool, + ) -> SupportedEffortTiers { + SupportedEffortTiers { + minimal, + low, + medium, + high, + xhigh, + max, + } + } + + fn replayed_search_turn(results: Value) -> Value { + json!([ + {"role": "user", "content": "when was Rome founded?"}, + {"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "when"}}, + {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": results}, + {"type": "text", "text": "753 BC."} + ]} + ]) + } + + #[fixture] + fn unmapped() -> AnthropicModelCapabilities { + AnthropicModelCapabilities::default() + } + + #[rstest] + #[case::empty_text(json!({"type": "thinking", "thinking": ""}), true)] + #[case::whitespace_only(json!({"type": "thinking", "thinking": " \n\t "}), true)] + #[case::null_text(json!({"type": "thinking", "thinking": null}), true)] + #[case::missing_text(json!({"type": "thinking"}), true)] + #[case::empty_text_despite_signature(json!({"type": "thinking", "thinking": "", "signature": "sig_abc"}), true)] + #[case::real_thinking(json!({"type": "thinking", "thinking": "plan", "signature": "sig"}), false)] + #[case::padded_real_thinking(json!({"type": "thinking", "thinking": " plan "}), false)] + #[case::redacted_thinking_is_a_different_type(json!({"type": "redacted_thinking", "data": "opaque"}), false)] + #[case::empty_text_block(json!({"type": "text", "text": ""}), false)] + #[case::untyped_block(json!({"thinking": ""}), false)] + fn empty_thinking_block_detection(#[case] input: Value, #[case] expected: bool) { + assert_eq!(is_empty_thinking_block(&block(input)), expected); + } + + #[rstest] + #[case::empty_text_beside_tool_use( + json!([{"role": "assistant", "content": [ + {"type": "text", "text": ""}, + {"type": "tool_use", "id": "x", "name": "Bash", "input": {}} + ]}]), + json!([{"role": "assistant", "content": [{"type": "tool_use", "id": "x", "name": "Bash", "input": {}}]}]) + )] + #[case::whitespace_text_beside_tool_use( + json!([{"role": "assistant", "content": [ + {"type": "text", "text": " \n "}, + {"type": "tool_use", "id": "x", "name": "Bash", "input": {}} + ]}]), + json!([{"role": "assistant", "content": [{"type": "tool_use", "id": "x", "name": "Bash", "input": {}}]}]) + )] + #[case::null_text( + json!([{"role": "user", "content": [ + {"type": "text", "text": null}, + {"type": "tool_result", "tool_use_id": "x", "content": "y"} + ]}]), + json!([{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "x", "content": "y"}]}]) + )] + #[case::missing_text( + json!([{"role": "user", "content": [ + {"type": "text"}, + {"type": "tool_result", "tool_use_id": "x", "content": "y"} + ]}]), + json!([{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "x", "content": "y"}]}]) + )] + #[case::empty_signed_thinking_beside_tool_use( + json!([ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "", "signature": "sig_abc"}, + {"type": "tool_use", "id": "toolu_01A", "name": "get_weather", "input": {"city": "Paris"}} + ]} + ]), + json!([ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "toolu_01A", "name": "get_weather", "input": {"city": "Paris"}} + ]} + ]) + )] + #[case::whitespace_thinking_beside_real_and_redacted_thinking( + json!([{"role": "assistant", "content": [ + {"type": "thinking", "thinking": " \n "}, + {"type": "thinking", "thinking": "real plan", "signature": "sig"}, + {"type": "redacted_thinking", "data": "opaque"} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "thinking", "thinking": "real plan", "signature": "sig"}, + {"type": "redacted_thinking", "data": "opaque"} + ]}]) + )] + #[case::blank_text_beside_real_thinking( + json!([{"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": "sig"}, + {"type": "text", "text": ""} + ]}]), + json!([{"role": "assistant", "content": [{"type": "thinking", "thinking": "plan", "signature": "sig"}]}]) + )] + #[case::message_left_without_blocks_is_dropped( + json!([ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": [{"type": "text", "text": ""}]}, + {"role": "assistant", "content": [{"type": "thinking", "thinking": ""}]} + ]), + json!([{"role": "user", "content": "hello"}]) + )] + fn strip_empty_content_blocks_rewrites(#[case] input: Value, #[case] expected: Value) { + assert_eq!(apply(strip_empty_content_blocks, input), expected); + } + + #[rstest] + #[case::non_empty_text(json!([{"role": "assistant", "content": [{"type": "text", "text": "hi"}]}]))] + #[case::padded_text(json!([{"role": "assistant", "content": [{"type": "text", "text": " hi "}]}]))] + #[case::empty_string_content(json!([{"role": "user", "content": ""}]))] + #[case::textless_non_text_block(json!([{"role": "user", "content": [ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AA=="}} + ]}]))] + #[case::encrypted_reasoning_left_for_the_responses_bridge(json!([{"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": tagged("gAAAA_1")}, + {"type": "redacted_thinking", "data": tagged("gAAAA_2")}, + {"type": "text", "text": "The answer."} + ]}]))] + fn strip_empty_content_blocks_leaves_untouched(#[case] input: Value) { + assert_eq!(apply(strip_empty_content_blocks, input.clone()), input); + } + + #[rstest] + #[case::replayed_provider_id("functions.Bash:0", "functions_Bash_0")] + #[case::thought_signature_suffix("call_abc123__thought__CiIBDDnWx+/a==", "call_abc123")] + #[case::splits_at_first_thought_separator("call_1__thought__a__thought__b", "call_1")] + #[case::valid_id("toolu_01-A_b", "toolu_01-A_b")] + #[case::non_ascii_letter("café", "caf_")] + #[case::only_invalid_characters("::", "__")] + #[case::empty("", "tool_use_id")] + #[case::thought_signature_only("__thought__CiIB", "tool_use_id")] + fn normalize_anthropic_tool_use_id_cases(#[case] raw: &str, #[case] expected: &str) { + assert_eq!(normalize_anthropic_tool_use_id(raw), expected); + } + + #[rstest] + #[case::tool_use_and_its_result( + json!([ + {"role": "assistant", "content": [{"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "functions.Bash:0", "content": "ok"}]} + ]), + json!([ + {"role": "assistant", "content": [{"type": "tool_use", "id": "functions_Bash_0", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "functions_Bash_0", "content": "ok"}]} + ]) + )] + #[case::server_tool_use( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "srv.1", "name": "web_search", "input": {}} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "srv_1", "name": "web_search", "input": {}} + ]}]) + )] + #[case::tool_use_rewrites_only_its_id( + json!([{"role": "assistant", "content": [ + {"type": "tool_use", "id": "a.b", "tool_use_id": "c.d", "name": "Bash", "input": {}} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "tool_use", "id": "a_b", "tool_use_id": "c.d", "name": "Bash", "input": {}} + ]}]) + )] + #[case::tool_result_rewrites_only_its_tool_use_id( + json!([{"role": "user", "content": [ + {"type": "tool_result", "id": "a.b", "tool_use_id": "c.d", "content": "ok"} + ]}]), + json!([{"role": "user", "content": [ + {"type": "tool_result", "id": "a.b", "tool_use_id": "c_d", "content": "ok"} + ]}]) + )] + fn sanitize_tool_use_ids_rewrites(#[case] input: Value, #[case] expected: Value) { + assert_eq!(apply(sanitize_tool_use_ids, input), expected); + } + + #[rstest] + #[case::valid_ids(json!([ + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "ok"}]} + ]))] + #[case::id_mentioned_in_text(json!([{"role": "user", "content": [{"type": "text", "text": "id: functions.Bash:0"}]}]))] + #[case::tool_use_without_id(json!([{"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {}}]}]))] + #[case::tool_result_without_tool_use_id(json!([{"role": "user", "content": [{"type": "tool_result", "content": "ok"}]}]))] + #[case::string_content(json!([{"role": "user", "content": "functions.Bash:0"}]))] + fn sanitize_tool_use_ids_leaves_untouched(#[case] input: Value) { + assert_eq!(apply(sanitize_tool_use_ids, input.clone()), input); + } + + #[rstest] + #[case::thinking_block( + json!([{"role": "assistant", "content": [ + {"type": "thinking", "thinking": "hm", "signature": "s", "provider_specific_fields": {"a": 1}} + ]}]), + json!([{"role": "assistant", "content": [{"type": "thinking", "thinking": "hm", "signature": "s"}]}]) + )] + #[case::every_block_of_every_message( + json!([ + {"role": "assistant", "content": [ + {"type": "text", "text": "a", "provider_specific_fields": {"x": 1}}, + {"type": "tool_use", "id": "t1", "name": "f", "input": {}, "provider_specific_fields": {"y": 2}} + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "ok", "provider_specific_fields": {}} + ]} + ]), + json!([ + {"role": "assistant", "content": [ + {"type": "text", "text": "a"}, + {"type": "tool_use", "id": "t1", "name": "f", "input": {}} + ]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "ok"}]} + ]) + )] + fn strip_provider_specific_fields_rewrites(#[case] input: Value, #[case] expected: Value) { + assert_eq!(apply(strip_provider_specific_fields, input), expected); + } + + #[rstest] + #[case::string_content(json!([{"role": "user", "content": "provider_specific_fields"}]))] + #[case::blocks_without_the_field(json!([{"role": "assistant", "content": [{"type": "text", "text": "a"}]}]))] + fn strip_provider_specific_fields_leaves_untouched(#[case] input: Value) { + assert_eq!(apply(strip_provider_specific_fields, input.clone()), input); + } + + #[rstest] + #[case::tagged_thinking_signature(json!({"type": "thinking", "thinking": "x", "signature": tagged("g")}), true)] + #[case::tagged_redacted_data(json!({"type": "redacted_thinking", "data": tagged("g")}), true)] + #[case::bare_tag_signature(json!({"type": "thinking", "thinking": "x", "signature": tagged("")}), true)] + #[case::bare_tag_data(json!({"type": "redacted_thinking", "data": tagged("")}), true)] + #[case::anthropic_signature(json!({"type": "thinking", "thinking": "x", "signature": "ErcBCkgIValid"}), false)] + #[case::anthropic_data(json!({"type": "redacted_thinking", "data": "EmwKAhgBEgy"}), false)] + #[case::unsigned_thinking(json!({"type": "thinking", "thinking": "x"}), false)] + #[case::tag_in_text_block(json!({"type": "text", "text": tagged("g")}), false)] + #[case::tag_in_thinking_data(json!({"type": "thinking", "thinking": "x", "data": tagged("g")}), false)] + #[case::tag_in_redacted_signature( + json!({"type": "redacted_thinking", "data": "EmwKAhgBEgy", "signature": tagged("g")}), + false + )] + #[case::tag_not_at_start(json!({"type": "thinking", "thinking": "x", "signature": format!("x{}", tagged("g"))}), false)] + fn encrypted_reasoning_block_detection(#[case] input: Value, #[case] expected: bool) { + assert_eq!(is_encrypted_reasoning_block(&block(input)), expected); + } + + #[rstest] + #[case::only_the_bridge_tagged_blocks( + json!([ + {"role": "user", "content": "Solve it."}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": tagged("gAAAA_1")}, + {"type": "redacted_thinking", "data": tagged("gAAAA_2")} + ]}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": tagged("gAAAA_3")}, + {"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."} + ]} + ]), + json!([ + {"role": "user", "content": "Solve it."}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "native", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."} + ]} + ]) + )] + #[case::bridge_turn_keeps_its_text( + json!([ + {"role": "user", "content": "Solve it."}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": tagged("gAAAA_1")}, + {"type": "redacted_thinking", "data": tagged("gAAAA_2")}, + {"type": "text", "text": "The answer."} + ]}, + {"role": "user", "content": "And the next one?"} + ]), + json!([ + {"role": "user", "content": "Solve it."}, + {"role": "assistant", "content": [{"type": "text", "text": "The answer."}]}, + {"role": "user", "content": "And the next one?"} + ]) + )] + fn strip_encrypted_reasoning_blocks_rewrites(#[case] input: Value, #[case] expected: Value) { + assert_eq!(apply(strip_encrypted_reasoning_blocks, input), expected); + } + + #[rstest] + #[case::anthropic_signed_blocks(json!([ + {"role": "user", "content": "Solve it."}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."} + ]} + ]))] + #[case::string_content(json!([{"role": "user", "content": tagged("g")}]))] + fn strip_encrypted_reasoning_blocks_leaves_untouched(#[case] input: Value) { + assert_eq!( + apply(strip_encrypted_reasoning_blocks, input.clone()), + input + ); + } + + #[rstest] + #[case::advisor_exchange_between_texts( + json!([ + {"role": "user", "content": "Build a worker pool."}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Let me consult the advisor."}, + {"type": "server_tool_use", "id": "srvtoolu_abc123", "name": "advisor", "input": {}}, + {"type": "advisor_tool_result", "tool_use_id": "srvtoolu_abc123", + "content": {"type": "advisor_result", "text": "Use channels."}}, + {"type": "text", "text": "Here is the implementation."} + ]} + ]), + json!([ + {"role": "user", "content": "Build a worker pool."}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Let me consult the advisor."}, + {"type": "text", "text": "Here is the implementation."} + ]} + ]) + )] + #[case::only_results_of_this_turns_advisor_calls( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "adv_1", "name": "advisor", "input": {}}, + {"type": "advisor_tool_result", "tool_use_id": "adv_1", "content": "advice"}, + {"type": "advisor_tool_result", "tool_use_id": "other", "content": "kept"}, + {"type": "tool_result", "tool_use_id": "adv_1", "content": "kept"}, + {"type": "text", "text": "answer"} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "advisor_tool_result", "tool_use_id": "other", "content": "kept"}, + {"type": "tool_result", "tool_use_id": "adv_1", "content": "kept"}, + {"type": "text", "text": "answer"} + ]}]) + )] + #[case::advisor_call_without_result( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "adv_1", "name": "advisor", "input": {}}, + {"type": "text", "text": "answer"} + ]}]), + json!([{"role": "assistant", "content": [{"type": "text", "text": "answer"}]}]) + )] + #[case::advisor_only_turn_keeps_an_empty_block_list( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "adv_1", "name": "advisor", "input": {}}, + {"type": "advisor_tool_result", "tool_use_id": "adv_1", "content": "advice"} + ]}]), + json!([{"role": "assistant", "content": []}]) + )] + fn strip_advisor_blocks_rewrites(#[case] input: Value, #[case] expected: Value) { + assert_eq!(apply(strip_advisor_blocks, input), expected); + } + + #[rstest] + #[case::no_advisor_blocks(json!([ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Hi there"}, + {"type": "tool_use", "id": "toolu_abc", "name": "get_weather", "input": {"location": "SF"}} + ]} + ]))] + #[case::user_turn(json!([{"role": "user", "content": [ + {"type": "server_tool_use", "id": "adv_2", "name": "advisor", "input": {}}, + {"type": "advisor_tool_result", "tool_use_id": "adv_2", "content": "advice"} + ]}]))] + #[case::other_server_tool(json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {}}, + {"type": "advisor_tool_result", "tool_use_id": "s1", "content": "advice"} + ]}]))] + #[case::client_tool_named_advisor(json!([{"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "advisor", "input": {}}, + {"type": "advisor_tool_result", "tool_use_id": "t1", "content": "advice"} + ]}]))] + #[case::advisor_call_with_empty_id(json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "", "name": "advisor", "input": {}}, + {"type": "advisor_tool_result", "tool_use_id": "", "content": "advice"} + ]}]))] + #[case::advisor_call_without_id(json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "name": "advisor", "input": {}} + ]}]))] + #[case::string_content(json!([{"role": "assistant", "content": "advisor"}]))] + fn strip_advisor_blocks_leaves_untouched(#[case] input: Value) { + assert_eq!(apply(strip_advisor_blocks, input.clone()), input); + } + + #[rstest] + #[case::results_keep_their_evidence( + json!([ + {"role": "user", "content": "latest version?"}, + {"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "latest version"}}, + {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": [ + {"type": "web_search_result", "url": "https://example.com/releases", "title": "Releases", + "page_age": null, "encrypted_content": "", "snippet": "Latest release v1.95.0"} + ]}, + {"type": "text", "text": "v1.95.0"} + ]} + ]), + json!([ + {"role": "user", "content": "latest version?"}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Web search results for 'latest version':\n\nTitle: Releases\nURL: https://example.com/releases\nSnippet: Latest release v1.95.0"}, + {"type": "text", "text": "v1.95.0"} + ]} + ]) + )] + #[case::each_result_lists_only_its_present_fields( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "q"}}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [ + {"type": "web_search_result", "title": "A"}, + {"type": "web_search_result", "snippet": "b"}, + {"type": "web_search_result", "url": "https://c"} + ]} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results for 'q':\n\nTitle: A\n\nSnippet: b\n\nURL: https://c"} + ]}]) + )] + #[case::result_without_fields_renders_the_header_only( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "q"}}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [{"type": "web_search_result"}]} + ]}]), + json!([{"role": "assistant", "content": [{"type": "text", "text": "Web search results for 'q':"}]}]) + )] + #[case::resultless_search( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "who won"}}, + {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": []}, + {"type": "text", "text": "I could not find that."} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results for 'who won':\n\nNo results were returned."}, + {"type": "text", "text": "I could not find that."} + ]}]) + )] + #[case::failed_search( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "q"}}, + {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", + "content": {"type": "web_search_tool_result_error", "error_code": "max_uses_exceeded"}} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results for 'q':\n\nSearch failed: max_uses_exceeded"} + ]}]) + )] + #[case::failed_search_without_error_code( + json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "e1", "content": {"type": "web_search_tool_result_error"}} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results:\n\nSearch failed: unavailable"} + ]}]) + )] + #[case::server_tool_use_without_query( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search"}, + {"type": "server_tool_use", "id": "s2", "name": "web_search", "input": {}}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": []}, + {"type": "web_search_tool_result", "tool_use_id": "s2", "content": []} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results:\n\nNo results were returned."}, + {"type": "text", "text": "Web search results:\n\nNo results were returned."} + ]}]) + )] + #[case::genuine_results_in_the_same_turn_stay( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "rust"}}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [ + {"type": "web_search_result", "url": "https://r", "title": "Rust", "snippet": "fast"} + ]}, + {"type": "server_tool_use", "id": "s2", "name": "web_search", "input": {"query": "real"}}, + {"type": "web_search_tool_result", "tool_use_id": "s2", "content": [ + {"type": "web_search_result", "url": "https://a", "title": "A", "snippet": "b", "encrypted_content": "enc"} + ]}, + {"type": "text", "text": "done"} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results for 'rust':\n\nTitle: Rust\nURL: https://r\nSnippet: fast"}, + {"type": "server_tool_use", "id": "s2", "name": "web_search", "input": {"query": "real"}}, + {"type": "web_search_tool_result", "tool_use_id": "s2", "content": [ + {"type": "web_search_result", "url": "https://a", "title": "A", "snippet": "b", "encrypted_content": "enc"} + ]}, + {"type": "text", "text": "done"} + ]}]) + )] + #[case::other_blocks_sharing_the_tool_use_id_stay( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "q"}}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": []}, + {"type": "tool_result", "tool_use_id": "s1", "content": "x"} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results for 'q':\n\nNo results were returned."}, + {"type": "tool_result", "tool_use_id": "s1", "content": "x"} + ]}]) + )] + #[case::query_lookup_stays_within_the_message( + json!([ + {"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "q"}} + ]}, + {"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": []} + ]} + ]), + json!([ + {"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "q"}} + ]}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Web search results:\n\nNo results were returned."} + ]} + ]) + )] + #[case::result_without_any_field_keeps_its_slot( + json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [ + {"type": "web_search_result", "url": "https://a", "title": "A"}, + {"type": "web_search_result"}, + {"type": "web_search_result", "title": "B"} + ]} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results:\n\nTitle: A\nURL: https://a\n\n\n\nTitle: B"} + ]}]) + )] + #[case::non_string_query_keeps_its_server_tool_use( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": 123}}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": []} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": 123}}, + {"type": "text", "text": "Web search results:\n\nNo results were returned."} + ]}]) + )] + #[case::non_object_input_keeps_its_server_tool_use( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": "q"}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": []} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": "q"}, + {"type": "text", "text": "Web search results:\n\nNo results were returned."} + ]}]) + )] + #[case::repeated_tool_use_id_renders_each_block_from_its_own_results( + json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": []}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": {"type": "web_search_tool_result_error", "error_code": "max_uses"}} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results:\n\nNo results were returned."}, + {"type": "text", "text": "Web search results:\n\nSearch failed: max_uses"} + ]}]) + )] + #[case::encrypted_block_sharing_a_replayed_id_stays( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "q"}}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": []}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [ + {"type": "web_search_result", "url": "https://a", "encrypted_content": "enc"} + ]} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results for 'q':\n\nNo results were returned."}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [ + {"type": "web_search_result", "url": "https://a", "encrypted_content": "enc"} + ]} + ]}]) + )] + #[case::last_query_wins_for_a_repeated_server_tool_use_id( + json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "first"}}, + {"type": "server_tool_use", "id": "s1", "name": "web_search", "input": {"query": "second"}}, + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": []} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "text", "text": "Web search results for 'second':\n\nNo results were returned."} + ]}]) + )] + fn flatten_unencrypted_web_search_results_rewrites( + #[case] input: Value, + #[case] expected: Value, + ) { + assert_eq!( + apply(flatten_unencrypted_web_search_results, input), + expected + ); + } + + #[rstest] + #[case::anthropic_issued_results(json!([{"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "q"}}, + {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": [ + {"type": "web_search_result", "url": "https://example.com", "title": "Example", + "page_age": null, "encrypted_content": "EqgfCioIARgBIiQ4"} + ]} + ]}]))] + #[case::any_encrypted_result_marks_the_block_genuine(json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [ + {"type": "web_search_result", "url": "https://a", "encrypted_content": ""}, + {"type": "web_search_result", "url": "https://b", "encrypted_content": "enc"} + ]} + ]}]))] + #[case::result_without_tool_use_id(json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "content": []} + ]}]))] + #[case::foreign_item_in_results(json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [ + {"type": "web_search_result", "url": "https://a"}, + {"type": "text", "text": "x"} + ]} + ]}]))] + #[case::result_with_null_url(json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": [ + {"type": "web_search_result", "url": null, "title": "A"} + ]} + ]}]))] + #[case::string_result_content(json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": "oops"} + ]}]))] + #[case::object_content_that_is_not_an_error(json!([{"role": "assistant", "content": [ + {"type": "web_search_tool_result", "tool_use_id": "s1", "content": {"type": "web_search_result", "url": "https://a"}} + ]}]))] + #[case::string_content(json!([{"role": "assistant", "content": "web_search_tool_result"}]))] + fn flatten_unencrypted_web_search_results_leaves_untouched(#[case] input: Value) { + assert_eq!( + apply(flatten_unencrypted_web_search_results, input.clone()), + input + ); + } + + #[rstest] + #[case::with_results(json!([{"type": "web_search_result", "url": "u", "title": "Rome", "snippet": "s", "page_age": null}]))] + #[case::without_results(json!([]))] + fn flatten_unencrypted_web_search_results_is_idempotent(#[case] results: Value) { + let input = replayed_search_turn(results); + let once = apply(flatten_unencrypted_web_search_results, input.clone()); + let twice = apply(flatten_unencrypted_web_search_results, once.clone()); + assert_ne!(once, input); + assert_eq!(twice, once); + } + + #[rstest] + #[case::no_existing_header(None, "b", "b")] + #[case::empty_existing_header(Some(""), "b", "b")] + #[case::whitespace_existing_header(Some(" "), "b", "b")] + #[case::sorted_after_merge(Some("c,a"), "b", "a,b,c")] + #[case::already_present(Some("a,b"), "a", "a,b")] + #[case::trimmed_and_deduplicated(Some("b, a ,b"), "c", "a,b,c")] + #[case::blank_pieces_skipped(Some("a,,b"), "c", "a,b,c")] + fn beta_values_merge_sorted_and_deduplicated( + #[case] existing: Option<&str>, + #[case] new_beta: &str, + #[case] expected: &str, + ) { + assert_eq!( + join_beta_values(split_beta_values(existing).chain([new_beta.to_string()])), + expected + ); + } + + #[rstest] + #[case::raw_token("sk-ant-oat01-abc123", true)] + #[case::bearer_token("Bearer sk-ant-oat02-xyz789", true)] + #[case::bare_prefix(ANTHROPIC_OAUTH_TOKEN_PREFIX, true)] + #[case::api_key("sk-ant-api01-abc123", false)] + #[case::bearer_api_key("Bearer sk-ant-api01-abc123", false)] + #[case::empty("", false)] + #[case::uppercase_prefix("sk-ant-OAT01-abc123", false)] + #[case::shouting_prefix("SK-ANT-OAT01-abc123", false)] + #[case::lowercase_bearer("bearer sk-ant-oat01-abc123", false)] + #[case::bearer_stripped_once("Bearer Bearer sk-ant-oat01-abc123", false)] + #[case::prefix_not_at_start(" sk-ant-oat01-abc123", false)] + fn anthropic_oauth_key_detection(#[case] value: &str, #[case] expected: bool) { + assert_eq!(is_anthropic_oauth_key(value), expected); + } + + #[rstest] + #[case::regex_tool(Some(json!([{"type": ANTHROPIC_TOOL_SEARCH_TOOL_TYPES[0], "name": "tool_search_tool_regex"}])), true)] + #[case::bm25_tool(Some(json!([{"type": ANTHROPIC_TOOL_SEARCH_TOOL_TYPES[1], "name": "tool_search_tool_bm25"}])), true)] + #[case::after_other_tools( + Some(json!([{"name": "get_weather", "input_schema": {}}, {"type": ANTHROPIC_TOOL_SEARCH_TOOL_TYPES[1]}])), + true + )] + #[case::function_tool(Some(json!([{"type": "function", "function": {"name": "get_weather"}}])), false)] + #[case::name_without_type(Some(json!([{"name": ANTHROPIC_TOOL_SEARCH_TOOL_TYPES[0]}])), false)] + #[case::empty_tools(Some(json!([])), false)] + #[case::no_tools(None, false)] + fn tool_search_detection(#[case] input: Option, #[case] expected: bool) { + assert_eq!(is_tool_search_used(tools(input).as_deref()), expected); + } + + #[rstest] + #[case::advisor_tool(Some(json!([{"type": ANTHROPIC_ADVISOR_TOOL_TYPE, "name": "advisor"}])), true)] + #[case::after_other_tools(Some(json!([{"name": "f", "input_schema": {}}, {"type": ANTHROPIC_ADVISOR_TOOL_TYPE}])), true)] + #[case::tool_named_advisor(Some(json!([{"name": "advisor", "input_schema": {}}])), false)] + #[case::other_server_tool(Some(json!([{"type": "web_search_20250305", "name": "web_search"}])), false)] + #[case::empty_tools(Some(json!([])), false)] + #[case::no_tools(None, false)] + fn advisor_tool_detection(#[case] input: Option, #[case] expected: bool) { + assert_eq!(has_advisor_tool(tools(input).as_deref()), expected); + } + + #[rstest] + #[case::param_without_history(Some(json!({})), json!([]), true)] + #[case::param_with_unsigned_history( + Some(json!({"trigger": 1})), + json!([{"role": "assistant", "content": [{"type": "compaction", "content": "c"}]}]), + true + )] + #[case::signed_block(None, json!([{"role": "assistant", "content": [{"type": "compaction", "content": "c", "signature": "s"}]}]), true)] + #[case::signed_block_later_in_history( + None, + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": [{"type": "text", "text": "a"}, {"type": "compaction", "content": "c", "signature": "s"}]} + ]), + true + )] + #[case::unsigned_block(None, json!([{"role": "assistant", "content": [{"type": "compaction", "content": "c"}]}]), false)] + #[case::empty_signature(None, json!([{"role": "assistant", "content": [{"type": "compaction", "content": "c", "signature": ""}]}]), false)] + #[case::signed_non_compaction_block( + None, + json!([{"role": "assistant", "content": [{"type": "thinking", "thinking": "t", "signature": "s"}]}]), + false + )] + #[case::string_content(None, json!([{"role": "user", "content": "compaction"}]), false)] + #[case::neither(None, json!([]), false)] + fn native_compaction_beta_requirement( + #[case] compaction: Option, + #[case] messages: Value, + #[case] expected: bool, + ) { + assert_eq!( + requires_native_compaction_beta(compaction.as_ref(), &history(messages)), + expected + ); + } + + #[rstest] + #[case::low(EffortLevel::Low, "low")] + #[case::medium(EffortLevel::Medium, "medium")] + #[case::high(EffortLevel::High, "high")] + #[case::xhigh(EffortLevel::Xhigh, "xhigh")] + #[case::max(EffortLevel::Max, "max")] + fn effort_level_names_agree_across_str_parse_and_serde( + #[case] level: EffortLevel, + #[case] name: &str, + ) { + assert_eq!(level.as_str(), name); + assert_eq!(EffortLevel::parse(name), Some(level)); + assert_eq!(serde_json::to_value(level).unwrap(), json!(name)); + assert_eq!( + serde_json::from_value::(json!(name)).unwrap(), + level + ); + } + + #[rstest] + #[case::unknown("ultra")] + #[case::minimal_is_not_an_output_config_level("minimal")] + #[case::uppercase("HIGH")] + #[case::empty("")] + fn effort_level_parse_rejects(#[case] value: &str) { + assert_eq!(EffortLevel::parse(value), None); + } + + #[rstest] + #[case::minimal_only(tiers(true, false, false, false, false, false), [false, false, false, false, false])] + #[case::low_only(tiers(false, true, false, false, false, false), [true, false, false, false, false])] + #[case::medium_only(tiers(false, false, true, false, false, false), [false, true, false, false, false])] + #[case::high_only(tiers(false, false, false, true, false, false), [false, false, true, false, false])] + #[case::xhigh_only(tiers(false, false, false, false, true, false), [false, false, false, true, false])] + #[case::max_only(tiers(false, false, false, false, false, true), [false, false, false, false, true])] + fn supports_effort_tier_reads_the_matching_flag( + #[case] effort_tiers: SupportedEffortTiers, + #[case] expected: [bool; 5], + unmapped: AnthropicModelCapabilities, + ) { + let capabilities = AnthropicModelCapabilities { + effort_tiers, + ..unmapped + }; + assert_eq!( + ALL_LEVELS.map(|level| capabilities.supports_effort_tier(level)), + expected + ); + } + + #[rstest] + #[case::unmapped(false, false, false, SupportedEffortTiers::default(), false)] + #[case::reasoning_and_adaptive_thinking_alone( + true, + true, + false, + SupportedEffortTiers::default(), + false + )] + #[case::output_config_without_tiers(false, false, true, SupportedEffortTiers::default(), true)] + #[case::minimal_tier( + false, + false, + false, + tiers(true, false, false, false, false, false), + true + )] + #[case::low_tier( + false, + false, + false, + tiers(false, true, false, false, false, false), + true + )] + #[case::medium_tier( + false, + false, + false, + tiers(false, false, true, false, false, false), + true + )] + #[case::high_tier( + false, + false, + false, + tiers(false, false, false, true, false, false), + true + )] + #[case::xhigh_tier( + false, + false, + false, + tiers(false, false, false, false, true, false), + true + )] + #[case::max_tier( + false, + false, + false, + tiers(false, false, false, false, false, true), + true + )] + fn supports_effort_param_cases( + #[case] supports_reasoning: bool, + #[case] supports_adaptive_thinking: bool, + #[case] supports_output_config: bool, + #[case] effort_tiers: SupportedEffortTiers, + #[case] expected: bool, + unmapped: AnthropicModelCapabilities, + ) { + let capabilities = AnthropicModelCapabilities { + supports_reasoning, + supports_adaptive_thinking, + supports_output_config, + effort_tiers, + ..unmapped + }; + assert_eq!(capabilities.supports_effort_param(), expected); + } + + #[rstest] + #[case::max_on_adaptive_thinking_model(true, SupportedEffortTiers::default(), "max", None)] + #[case::max_on_max_tier_model( + false, + tiers(false, false, false, false, false, true), + "max", + None + )] + #[case::max_on_output_config_only_model( + false, + SupportedEffortTiers::default(), + "max", + Some("effort='max' is not supported by this model. Got model: claude-test") + )] + #[case::max_on_xhigh_tier_model( + false, + tiers(false, false, false, false, true, false), + "max", + Some("effort='max' is not supported by this model. Got model: claude-test") + )] + #[case::xhigh_on_xhigh_tier_model( + false, + tiers(false, false, false, false, true, false), + "xhigh", + None + )] + #[case::xhigh_on_adaptive_thinking_model( + true, + SupportedEffortTiers::default(), + "xhigh", + Some("effort='xhigh' is not supported by this model. Got model: claude-test") + )] + #[case::xhigh_on_max_tier_model( + false, + tiers(false, false, false, false, false, true), + "xhigh", + Some("effort='xhigh' is not supported by this model. Got model: claude-test") + )] + #[case::high_on_unmapped_model(false, SupportedEffortTiers::default(), "high", None)] + #[case::low_on_unmapped_model(false, SupportedEffortTiers::default(), "low", None)] + #[case::unknown_level_is_left_to_other_validation( + false, + SupportedEffortTiers::default(), + "ultra", + None + )] + fn effort_level_rejection_cases( + #[case] supports_adaptive_thinking: bool, + #[case] effort_tiers: SupportedEffortTiers, + #[case] effort: &str, + #[case] expected: Option<&str>, + unmapped: AnthropicModelCapabilities, + ) { + let capabilities = AnthropicModelCapabilities { + supports_output_config: true, + supports_adaptive_thinking, + effort_tiers, + ..unmapped + }; + assert_eq!( + capabilities + .effort_level_rejection(effort, "claude-test") + .as_deref(), + expected + ); + } + + #[rstest] + fn unmapped_model_has_no_reasoning_features_but_accepts_sampling_params( + unmapped: AnthropicModelCapabilities, + ) { + assert_eq!( + unmapped, + AnthropicModelCapabilities { + supports_reasoning: false, + supports_adaptive_thinking: false, + thinking_always_on: false, + supports_legacy_thinking: false, + supports_output_config: false, + supports_sampling_params: true, + supports_speed: false, + effort_tiers: tiers(false, false, false, false, false, false), + } + ); + assert_eq!( + serde_json::from_value::(json!({})).unwrap(), + unmapped + ); + } + + #[rstest] + #[case::sampling_params_removed( + json!({"supports_sampling_params": false}), + AnthropicModelCapabilities { supports_sampling_params: false, ..AnthropicModelCapabilities::default() } + )] + #[case::fast_mode( + json!({"supports_speed": true}), + AnthropicModelCapabilities { supports_speed: true, ..AnthropicModelCapabilities::default() } + )] + #[case::partial_effort_tiers( + json!({"supports_reasoning": true, "effort_tiers": {"xhigh": true}}), + AnthropicModelCapabilities { + supports_reasoning: true, + effort_tiers: tiers(false, false, false, false, true, false), + ..AnthropicModelCapabilities::default() + } + )] + fn capabilities_fill_missing_flags_with_unmapped_defaults( + #[case] input: Value, + #[case] expected: AnthropicModelCapabilities, + ) { + assert_eq!( + serde_json::from_value::(input).unwrap(), + expected + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/handler.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/handler.rs new file mode 100644 index 00000000000..0e2ab97956a --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/handler.rs @@ -0,0 +1,270 @@ +use litellm_types::llms::anthropic_messages::anthropic_request::{ + AnthropicMessage, AnthropicMessagesRequest, +}; +use serde_json::{Value, json}; + +use crate::{ + anthropic::common_utils::{ + flatten_unencrypted_web_search_results, sanitize_tool_use_ids, strip_empty_content_blocks, + strip_provider_specific_fields, + }, + base_llm::chat::transformation::Error, +}; + +pub fn shape_anthropic_messages_request( + request: AnthropicMessagesRequest, + reasoning_auto_summary: bool, +) -> Result { + Ok(AnthropicMessagesRequest { + messages: sanitize_anthropic_messages(request.messages), + metadata: request + .metadata + .as_ref() + .map(validate_anthropic_api_metadata) + .transpose()?, + thinking: with_reasoning_auto_summary(request.thinking, reasoning_auto_summary), + ..request + }) +} + +fn sanitize_anthropic_messages(messages: Vec) -> Vec { + strip_provider_specific_fields(flatten_unencrypted_web_search_results( + sanitize_tool_use_ids(strip_empty_content_blocks(messages)), + )) +} + +fn validate_anthropic_api_metadata(metadata: &Value) -> Result { + let Value::Object(fields) = metadata else { + return Err(Error::InvalidRequest(format!( + "metadata must be an object, got {metadata}" + ))); + }; + match fields.get("user_id") { + None | Some(Value::Null) => Ok(json!({})), + Some(Value::String(user_id)) => Ok(json!({"user_id": user_id})), + Some(other) => Err(Error::InvalidRequest(format!( + "metadata.user_id must be a string, got {other}" + ))), + } +} + +fn with_reasoning_auto_summary(thinking: Option, enabled: bool) -> Option { + let Some(Value::Object(thinking)) = thinking else { + return thinking; + }; + if !enabled || thinking.get("type").and_then(Value::as_str) == Some("disabled") { + return Some(Value::Object(thinking)); + } + Some(Value::Object( + thinking + .into_iter() + .filter(|(key, _)| key != "display") + .chain([("display".to_string(), json!("summarized"))]) + .collect(), + )) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn messages(value: Value) -> Vec { + serde_json::from_value(value).unwrap() + } + + fn request(body: Value) -> AnthropicMessagesRequest { + serde_json::from_value(body).unwrap() + } + + #[rstest] + #[case::empty_text_next_to_a_tool_use( + json!([{"role": "assistant", "content": [ + {"type": "text", "text": " "}, + {"type": "tool_use", "id": "t", "name": "B", "input": {}} + ]}]), + json!([{"role": "assistant", "content": [ + {"type": "tool_use", "id": "t", "name": "B", "input": {}} + ]}]), + )] + #[case::cross_provider_tool_ids( + json!([ + {"role": "assistant", "content": [{"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "functions.Bash:0", "content": "ok"}]} + ]), + json!([ + {"role": "assistant", "content": [{"type": "tool_use", "id": "functions_Bash_0", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "functions_Bash_0", "content": "ok"}]} + ]), + )] + #[case::replayed_unencrypted_web_search_results( + json!([ + {"role": "user", "content": "latest litellm version?"}, + {"role": "assistant", "content": [ + {"type": "server_tool_use", "id": "srvtoolu_1", "name": "web_search", "input": {"query": "latest litellm version"}}, + {"type": "web_search_tool_result", "tool_use_id": "srvtoolu_1", "content": [{ + "type": "web_search_result", + "url": "https://github.com/BerriAI/litellm/releases", + "title": "Releases", + "page_age": null, + "encrypted_content": "", + "snippet": "Latest release v1.95.0" + }]} + ]}, + {"role": "user", "content": "which version?"} + ]), + json!([ + {"role": "user", "content": "latest litellm version?"}, + {"role": "assistant", "content": [{ + "type": "text", + "text": "Web search results for 'latest litellm version':\n\nTitle: Releases\nURL: https://github.com/BerriAI/litellm/releases\nSnippet: Latest release v1.95.0" + }]}, + {"role": "user", "content": "which version?"} + ]), + )] + #[case::replayed_provider_specific_fields( + json!([ + {"role": "assistant", "content": [{ + "type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}, + "provider_specific_fields": {"signature": "sig_abc"} + }]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "Sunny"}]} + ]), + json!([ + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_01", "name": "get_weather", "input": {"city": "Paris"}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_01", "content": "Sunny"}]} + ]), + )] + #[case::ids_are_normalized_before_web_search_results_flatten( + json!([ + {"role": "user", "content": "run it"}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "", "signature": "sig"}, + {"type": "text", "text": ""}, + {"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}, "provider_specific_fields": {"x": 1}}, + {"type": "server_tool_use", "id": "srv.1", "name": "web_search", "input": {"query": "q"}, "provider_specific_fields": {"x": 2}}, + {"type": "web_search_tool_result", "tool_use_id": "srv.1", "provider_specific_fields": {"x": 3}, "content": [ + {"type": "web_search_result", "url": "u", "title": "", "encrypted_content": "", "provider_specific_fields": {"x": 4}} + ]} + ]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "functions.Bash:0", "content": "ok"}]}, + {"role": "assistant", "content": [{"type": "text", "text": " "}]} + ]), + json!([ + {"role": "user", "content": "run it"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "Bash", "input": {}}, + {"type": "server_tool_use", "id": "srv_1", "name": "web_search", "input": {"query": "q"}}, + {"type": "text", "text": "Web search results:\n\nURL: u"} + ]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "functions_Bash_0", "content": "ok"}]} + ]), + )] + fn sanitize_anthropic_messages_cleans_replayed_history( + #[case] history: Value, + #[case] expected: Value, + ) { + assert_eq!( + serde_json::to_value(sanitize_anthropic_messages(messages(history))).unwrap(), + expected + ); + } + + #[rstest] + #[case::keeps_only_user_id(json!({"user_id": "u-1", "trace_id": "internal"}), Ok(json!({"user_id": "u-1"})))] + #[case::null_user_id(json!({"user_id": null, "trace_id": "internal"}), Ok(json!({})))] + #[case::no_user_id(json!({"trace_id": "internal"}), Ok(json!({})))] + #[case::empty(json!({}), Ok(json!({})))] + #[case::numeric_user_id( + json!({"user_id": 123}), + Err(Error::InvalidRequest("metadata.user_id must be a string, got 123".to_string())), + )] + #[case::boolean_user_id( + json!({"user_id": true}), + Err(Error::InvalidRequest("metadata.user_id must be a string, got true".to_string())), + )] + #[case::not_an_object( + json!(["u-1"]), + Err(Error::InvalidRequest(r#"metadata must be an object, got ["u-1"]"#.to_string())), + )] + fn validate_anthropic_api_metadata_passes_only_a_string_user_id( + #[case] metadata: Value, + #[case] expected: Result, + ) { + assert_eq!(validate_anthropic_api_metadata(&metadata), expected); + } + + #[rstest] + #[case::adaptive( + Some(json!({"type": "adaptive", "budget_tokens": 5000})), + true, + Some(json!({"type": "adaptive", "budget_tokens": 5000, "display": "summarized"})), + )] + #[case::enabled( + Some(json!({"type": "enabled", "budget_tokens": 10000})), + true, + Some(json!({"type": "enabled", "budget_tokens": 10000, "display": "summarized"})), + )] + #[case::no_type(Some(json!({})), true, Some(json!({"display": "summarized"})))] + #[case::display_omitted_is_overridden( + Some(json!({"type": "enabled", "budget_tokens": 10000, "display": "omitted"})), + true, + Some(json!({"type": "enabled", "budget_tokens": 10000, "display": "summarized"})), + )] + #[case::display_summarized_is_kept( + Some(json!({"type": "enabled", "display": "summarized"})), + true, + Some(json!({"type": "enabled", "display": "summarized"})), + )] + #[case::disabled_thinking(Some(json!({"type": "disabled"})), true, Some(json!({"type": "disabled"})))] + #[case::flag_off( + Some(json!({"type": "enabled", "budget_tokens": 10000})), + false, + Some(json!({"type": "enabled", "budget_tokens": 10000})), + )] + #[case::flag_off_keeps_callers_display( + Some(json!({"type": "enabled", "display": "omitted"})), + false, + Some(json!({"type": "enabled", "display": "omitted"})), + )] + #[case::no_thinking(None, true, None)] + #[case::non_object_thinking(Some(json!("enabled")), true, Some(json!("enabled")))] + fn reasoning_auto_summary_marks_active_thinking_as_summarized( + #[case] thinking: Option, + #[case] enabled: bool, + #[case] expected: Option, + ) { + assert_eq!(with_reasoning_auto_summary(thinking, enabled), expected); + } + + #[test] + fn shaping_cleans_messages_metadata_and_thinking() { + let sanitized = shape_anthropic_messages_request( + request(json!({ + "model": "m", + "messages": [{"role": "assistant", "content": [ + {"type": "text", "text": ""}, + {"type": "tool_use", "id": "functions.Bash:0", "name": "Bash", "input": {}} + ]}], + "metadata": {"user_id": "u", "trace_id": "t"}, + "thinking": {"type": "enabled", "budget_tokens": 1024}, + "safeguards": [{"type": "dangerous_tool_use"}] + })), + true, + ) + .unwrap(); + assert_eq!( + serde_json::to_value(sanitized).unwrap(), + json!({ + "model": "m", + "messages": [{"role": "assistant", "content": [ + {"type": "tool_use", "id": "functions_Bash_0", "name": "Bash", "input": {}} + ]}], + "metadata": {"user_id": "u"}, + "thinking": {"type": "enabled", "budget_tokens": 1024, "display": "summarized"}, + "safeguards": [{"type": "dangerous_tool_use"}] + }) + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/headers.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/headers.rs new file mode 100644 index 00000000000..8d48d7a0f5c --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/headers.rs @@ -0,0 +1,643 @@ +use litellm_types::llms::anthropic_messages::anthropic_request::AnthropicMessagesRequest; +use serde_json::Value; + +use crate::{ + anthropic::{ + ANTHROPIC_OAUTH_TOKEN_PREFIX, + common_utils::{ + ANTHROPIC_OAUTH_BETA_HEADER, beta, has_advisor_tool, is_anthropic_oauth_key, + is_tool_search_used, join_beta_values, requires_native_compaction_beta, + split_beta_values, + }, + }, + base_llm::anthropic_messages::transformation::Headers, +}; + +const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; +const ANTHROPIC_AUTH_TOKEN_ENV: &str = "ANTHROPIC_AUTH_TOKEN"; +const BETA_HEADER: &str = "anthropic-beta"; +const AUTHORIZATION: &str = "authorization"; +const API_KEY_HEADER: &str = "x-api-key"; +const DIRECT_BROWSER_ACCESS_HEADER: &str = "anthropic-dangerous-direct-browser-access"; + +fn header_value<'a>(headers: &'a [(String, String)], name: &str) -> Option<&'a str> { + headers + .iter() + .find(|(header, _)| header.eq_ignore_ascii_case(name)) + .map(|(_, value)| value.as_str()) +} + +fn without(headers: Headers, names: &[&str]) -> Headers { + headers + .into_iter() + .filter(|(header, _)| !names.iter().any(|name| header.eq_ignore_ascii_case(name))) + .collect() +} + +fn existing_betas(headers: &[(String, String)]) -> impl Iterator + '_ { + headers + .iter() + .filter(|(header, _)| header.eq_ignore_ascii_case(BETA_HEADER)) + .flat_map(|(_, value)| split_beta_values(Some(value))) +} + +fn with_oauth_bearer(headers: Headers, bearer: String) -> Headers { + let beta = + join_beta_values(existing_betas(&headers).chain([ANTHROPIC_OAUTH_BETA_HEADER.to_string()])); + without(headers, &[API_KEY_HEADER, AUTHORIZATION, BETA_HEADER]) + .into_iter() + .chain([ + (AUTHORIZATION.to_string(), bearer), + (BETA_HEADER.to_string(), beta), + (DIRECT_BROWSER_ACCESS_HEADER.to_string(), "true".to_string()), + ]) + .collect() +} + +fn non_empty(value: Option<&str>) -> Option<&str> { + value.map(str::trim).filter(|value| !value.is_empty()) +} + +pub fn authenticate( + headers: Headers, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, +) -> Result { + if let Some(forwarded) = header_value(&headers, AUTHORIZATION) + && forwarded + .strip_prefix("Bearer ") + .is_some_and(|token| token.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX)) + { + let bearer = forwarded.to_string(); + return Ok(with_oauth_bearer(headers, bearer)); + } + if let Some(key) = api_key.filter(|key| key.starts_with(ANTHROPIC_OAUTH_TOKEN_PREFIX)) { + return Ok(with_oauth_bearer(headers, format!("Bearer {key}"))); + } + if header_value(&headers, API_KEY_HEADER).is_some() + || header_value(&headers, AUTHORIZATION).is_some() + { + return Ok(headers); + } + let resolved_key = non_empty(api_key) + .map(str::to_string) + .or_else(|| env_lookup(ANTHROPIC_API_KEY_ENV).filter(|value| !value.trim().is_empty())); + let auth = match resolved_key { + Some(key) if is_anthropic_oauth_key(&key) => { + (AUTHORIZATION.to_string(), format!("Bearer {key}")) + } + Some(key) => (API_KEY_HEADER.to_string(), key), + None => match env_lookup(ANTHROPIC_AUTH_TOKEN_ENV).filter(|value| !value.trim().is_empty()) + { + Some(token) => (AUTHORIZATION.to_string(), format!("Bearer {token}")), + None => { + return Err(litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: ANTHROPIC_API_KEY_ENV, + }); + } + }, + }; + Ok(headers.into_iter().chain([auth]).collect()) +} + +fn context_management_betas( + context_management: Option<&Value>, +) -> impl Iterator { + let edits = context_management + .and_then(|value| value.get("edits")) + .and_then(Value::as_array) + .map(Vec::as_slice) + .unwrap_or(&[]); + let (compact, other) = edits.iter().fold((false, false), |(compact, other), edit| { + match edit.get("type").and_then(Value::as_str) { + Some("compact_20260112") => (true, other), + _ => (compact, true), + } + }); + compact + .then_some(beta::COMPACT_2026_01_12) + .into_iter() + .chain(other.then_some(beta::CONTEXT_MANAGEMENT_2025_06_27)) +} + +fn uses_structured_output(request: &AnthropicMessagesRequest) -> bool { + request.output_format.is_some() + || request + .output_config + .as_ref() + .and_then(|config| config.get("format")) + .is_some_and(|format| !format.is_null()) +} + +fn messages_carry_output_config(request: &AnthropicMessagesRequest) -> bool { + request + .messages + .iter() + .any(|message| message.extra.contains_key("output_config")) +} + +pub fn feature_betas(request: &AnthropicMessagesRequest) -> Vec<&'static str> { + let tools = request.tools.as_deref(); + [ + requires_native_compaction_beta(request.compaction.as_ref(), &request.messages) + .then_some(beta::COMPACT_2026_09_04), + uses_structured_output(request).then_some(beta::STRUCTURED_OUTPUT), + (request.speed.as_deref() == Some("fast")).then_some(beta::FAST_MODE_2026_02_01), + messages_carry_output_config(request).then_some(beta::PER_TURN_CONTROL_2026_07_01), + has_advisor_tool(tools).then_some(beta::ADVISOR_TOOL_2026_03_01), + is_tool_search_used(tools).then_some(beta::ADVANCED_TOOL_USE_2025_11_20), + ] + .into_iter() + .flatten() + .chain(context_management_betas( + request.context_management.as_ref(), + )) + .collect() +} + +pub fn with_feature_betas(headers: Headers, request: &AnthropicMessagesRequest) -> Headers { + let existing = existing_betas(&headers).collect::>(); + let features = feature_betas(request); + if existing.is_empty() && features.is_empty() { + return headers; + } + let merged = join_beta_values( + existing + .into_iter() + .chain(features.into_iter().map(str::to_string)), + ); + without(headers, &[BETA_HEADER]) + .into_iter() + .chain([(BETA_HEADER.to_string(), merged)]) + .collect() +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + use serde_json::json; + + use super::*; + + const OAUTH_TOKEN: &str = "sk-ant-oat01-token"; + const OAUTH_BEARER: &str = "Bearer sk-ant-oat01-token"; + const REGULAR_KEY: &str = "sk-ant-api03-regular"; + const BROWSER_ACCESS: (&str, &str) = ("anthropic-dangerous-direct-browser-access", "true"); + + type Env = &'static [(&'static str, &'static str)]; + + fn request(fields: Value) -> AnthropicMessagesRequest { + let mut body = + json!({"model": "claude", "messages": [{"role": "user", "content": "Hello"}]}); + body.as_object_mut() + .unwrap() + .extend(fields.as_object().unwrap().clone()); + serde_json::from_value(body).unwrap() + } + + fn headers(pairs: &[(&str, &str)]) -> Headers { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect() + } + + fn betas(values: &[&str]) -> String { + values.join(",") + } + + #[fixture] + fn no_env() -> Env { + &[] + } + + #[fixture] + fn full_env() -> Env { + &[ + ("ANTHROPIC_API_KEY", "sk-env"), + ("ANTHROPIC_AUTH_TOKEN", "env-token"), + ] + } + + fn authenticate_with( + forwarded: &[(&str, &str)], + api_key: Option<&str>, + env: Env, + ) -> Result { + let lookup = |name: &str| { + env.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + }; + authenticate(headers(forwarded), api_key, &lookup) + } + + #[rstest] + #[case::forwarded_bearer_drops_forwarded_and_deployment_keys( + &[("X-Api-Key", REGULAR_KEY), ("Authorization", OAUTH_BEARER)], + Some(REGULAR_KEY), + OAUTH_BEARER, + &[], + )] + #[case::forwarded_bearer_in_uppercase_authorization_header( + &[("AUTHORIZATION", OAUTH_BEARER)], + None, + OAUTH_BEARER, + &[], + )] + #[case::forwarded_bearer_keeps_unrelated_headers_in_place( + &[("anthropic-version", "2023-06-01"), ("authorization", OAUTH_BEARER)], + None, + OAUTH_BEARER, + &[("anthropic-version", "2023-06-01")], + )] + #[case::forwarded_bearer_wins_over_an_oauth_api_key( + &[("authorization", OAUTH_BEARER)], + Some("sk-ant-oat01-deployment"), + OAUTH_BEARER, + &[], + )] + #[case::api_key_authenticates_as_a_bearer(&[], Some(OAUTH_TOKEN), OAUTH_BEARER, &[])] + #[case::api_key_removes_a_forwarded_x_api_key( + &[("x-api-key", OAUTH_TOKEN)], + Some(OAUTH_TOKEN), + OAUTH_BEARER, + &[], + )] + #[case::api_key_replaces_a_forwarded_non_oauth_bearer( + &[("Authorization", "Bearer some-proxy-token")], + Some(OAUTH_TOKEN), + OAUTH_BEARER, + &[], + )] + fn oauth_token_is_the_whole_credential( + #[case] forwarded: &[(&str, &str)], + #[case] api_key: Option<&str>, + #[case] expected_bearer: &str, + #[case] kept: &[(&str, &str)], + full_env: Env, + ) { + let expected = kept + .iter() + .copied() + .chain([ + ("authorization", expected_bearer), + ("anthropic-beta", ANTHROPIC_OAUTH_BETA_HEADER), + BROWSER_ACCESS, + ]) + .collect::>(); + assert_eq!( + authenticate_with(forwarded, api_key, full_env).unwrap(), + headers(&expected) + ); + } + + #[rstest] + #[case::forwarded_bearer_merges_a_differently_cased_beta_header( + &[("Anthropic-Beta", "web-search-2025-03-05"), ("authorization", OAUTH_BEARER)], + None, + )] + #[case::forwarded_bearer_dedupes_an_existing_oauth_beta( + &[("anthropic-beta", "web-search-2025-03-05, oauth-2025-04-20"), ("authorization", OAUTH_BEARER)], + None, + )] + #[case::api_key_merges_the_existing_beta_header( + &[("anthropic-beta", " web-search-2025-03-05 ,")], + Some(OAUTH_TOKEN), + )] + #[case::forwarded_bearer_unions_every_beta_header_casing( + &[("anthropic-beta", "oauth-2025-04-20"), ("ANTHROPIC-BETA", "web-search-2025-03-05"), ("authorization", OAUTH_BEARER)], + None, + )] + fn oauth_beta_merges_into_existing_betas( + #[case] forwarded: &[(&str, &str)], + #[case] api_key: Option<&str>, + no_env: Env, + ) { + assert_eq!( + authenticate_with(forwarded, api_key, no_env).unwrap(), + headers(&[ + ("authorization", OAUTH_BEARER), + ( + "anthropic-beta", + &betas(&[ANTHROPIC_OAUTH_BETA_HEADER, "web-search-2025-03-05"]) + ), + BROWSER_ACCESS, + ]) + ); + } + + #[rstest] + #[case::x_api_key_over_the_deployment_key(&[("x-api-key", "caller-key")], Some("sk-other"))] + #[case::uppercase_x_api_key(&[("X-API-KEY", "caller-key")], None)] + #[case::non_oauth_bearer(&[("Authorization", "Bearer some-proxy-token")], None)] + #[case::non_oauth_bearer_over_a_regular_api_key( + &[("authorization", "Bearer sk-ant-api03-forwarded")], + Some(REGULAR_KEY), + )] + #[case::oauth_token_without_the_bearer_scheme(&[("authorization", OAUTH_TOKEN)], None)] + #[case::oauth_token_behind_a_lowercase_bearer_scheme( + &[("authorization", "bearer sk-ant-oat01-token")], + None, + )] + fn forwarded_auth_header_is_kept_untouched( + #[case] forwarded: &[(&str, &str)], + #[case] api_key: Option<&str>, + full_env: Env, + ) { + assert_eq!( + authenticate_with(forwarded, api_key, full_env).unwrap(), + headers(forwarded) + ); + } + + #[rstest] + #[case::api_key_param(Some("sk-param"), &[], ("x-api-key", "sk-param"))] + #[case::api_key_param_over_env_key_and_auth_token( + Some("sk-param"), + &[("ANTHROPIC_API_KEY", "sk-env"), ("ANTHROPIC_AUTH_TOKEN", "env-token")], + ("x-api-key", "sk-param"), + )] + #[case::env_key_without_a_param(None, &[("ANTHROPIC_API_KEY", "sk-env")], ("x-api-key", "sk-env"))] + #[case::env_key_when_the_param_is_empty(Some(""), &[("ANTHROPIC_API_KEY", "sk-env")], ("x-api-key", "sk-env"))] + #[case::env_key_when_the_param_is_whitespace( + Some(" "), + &[("ANTHROPIC_API_KEY", "sk-env")], + ("x-api-key", "sk-env"), + )] + #[case::env_key_over_auth_token( + None, + &[("ANTHROPIC_API_KEY", "sk-env"), ("ANTHROPIC_AUTH_TOKEN", "env-token")], + ("x-api-key", "sk-env"), + )] + #[case::auth_token_as_a_bearer( + None, + &[("ANTHROPIC_AUTH_TOKEN", "env-token")], + ("authorization", "Bearer env-token"), + )] + #[case::auth_token_when_the_env_key_is_whitespace( + None, + &[("ANTHROPIC_API_KEY", " \t"), ("ANTHROPIC_AUTH_TOKEN", "env-token")], + ("authorization", "Bearer env-token"), + )] + #[case::oauth_env_key_as_a_plain_bearer( + None, + &[("ANTHROPIC_API_KEY", "sk-ant-oat01-env")], + ("authorization", "Bearer sk-ant-oat01-env"), + )] + fn credential_is_resolved_after_the_existing_headers( + #[case] api_key: Option<&str>, + #[case] env: Env, + #[case] expected: (&str, &str), + ) { + let forwarded = [("anthropic-beta", "web-search-2025-03-05")]; + assert_eq!( + authenticate_with(&forwarded, api_key, env).unwrap(), + headers(&[forwarded[0], expected]) + ); + } + + #[rstest] + #[case::no_credentials(&[], None, &[])] + #[case::empty_api_key(&[], Some(""), &[])] + #[case::whitespace_only_env_values( + &[], + None, + &[("ANTHROPIC_API_KEY", " "), ("ANTHROPIC_AUTH_TOKEN", " \t")], + )] + #[case::unrelated_forwarded_headers(&[("anthropic-beta", "web-search-2025-03-05")], None, &[])] + fn missing_credentials_are_an_auth_error( + #[case] forwarded: &[(&str, &str)], + #[case] api_key: Option<&str>, + #[case] env: Env, + ) { + assert!(matches!( + authenticate_with(forwarded, api_key, env), + Err(litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: "ANTHROPIC_API_KEY", + }) + )); + } + + #[rstest] + #[case::no_features(json!({}), &[])] + #[case::output_format(json!({"output_format": {"type": "json_schema"}}), &[beta::STRUCTURED_OUTPUT])] + #[case::null_output_format(json!({"output_format": null}), &[])] + #[case::output_config_format( + json!({"output_config": {"format": {"type": "json_schema"}, "effort": "xhigh"}}), + &[beta::STRUCTURED_OUTPUT] + )] + #[case::null_output_config_format(json!({"output_config": {"format": null}}), &[])] + #[case::top_level_output_config_without_format(json!({"output_config": {"effort": "high"}}), &[])] + #[case::fast_speed(json!({"speed": "fast"}), &[beta::FAST_MODE_2026_02_01])] + #[case::standard_speed(json!({"speed": "standard"}), &[])] + #[case::compaction_param(json!({"compaction": {"enabled": true}}), &[beta::COMPACT_2026_09_04])] + #[case::empty_compaction_param(json!({"compaction": {}}), &[beta::COMPACT_2026_09_04])] + #[case::signed_compaction_block_in_history( + json!({"messages": [ + {"role": "assistant", "content": [{"type": "compaction", "content": "summary", "signature": "sig"}]}, + {"role": "user", "content": "Continue"}, + ]}), + &[beta::COMPACT_2026_09_04] + )] + #[case::unsigned_compaction_block_in_history( + json!({"messages": [ + {"role": "assistant", "content": [{"type": "compaction", "content": "summary", "signature": ""}]}, + {"role": "user", "content": "Continue"}, + ]}), + &[] + )] + #[case::advisor_tool( + json!({"tools": [{"type": "advisor_20260301", "name": "advisor", "model": "claude-opus-4-6"}]}), + &[beta::ADVISOR_TOOL_2026_03_01] + )] + #[case::no_tools(json!({"tools": []}), &[])] + #[case::regex_tool_search( + json!({"tools": [{"type": "tool_search_tool_regex_20251119"}]}), + &[beta::ADVANCED_TOOL_USE_2025_11_20] + )] + #[case::bm25_tool_search( + json!({"tools": [{"type": "tool_search_tool_bm25_20251119"}]}), + &[beta::ADVANCED_TOOL_USE_2025_11_20] + )] + #[case::unrelated_server_tool(json!({"tools": [{"type": "web_search_20250305", "name": "web_search"}]}), &[])] + #[case::only_compact_edits( + json!({"context_management": {"edits": [{"type": "compact_20260112"}]}}), + &[beta::COMPACT_2026_01_12] + )] + #[case::only_other_edits( + json!({"context_management": {"edits": [{"type": "clear_tool_uses_20250919", "keep": {"type": "tool_uses", "value": 3}}]}}), + &[beta::CONTEXT_MANAGEMENT_2025_06_27] + )] + #[case::compact_and_other_edits( + json!({"context_management": {"edits": [{"type": "compact_20260112"}, {"type": "clear_tool_uses_20250919"}]}}), + &[beta::COMPACT_2026_01_12, beta::CONTEXT_MANAGEMENT_2025_06_27] + )] + #[case::edit_without_a_type(json!({"context_management": {"edits": [{}]}}), &[beta::CONTEXT_MANAGEMENT_2025_06_27])] + #[case::empty_edits(json!({"context_management": {"edits": []}}), &[])] + #[case::context_management_without_edits(json!({"context_management": {}}), &[])] + #[case::per_message_output_config( + json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}), + &[beta::PER_TURN_CONTROL_2026_07_01] + )] + #[case::per_message_null_output_config( + json!({"messages": [{"role": "user", "content": "hi", "output_config": null}]}), + &[beta::PER_TURN_CONTROL_2026_07_01] + )] + fn feature_betas_follow_the_request(#[case] fields: Value, #[case] expected: &[&str]) { + assert_eq!(feature_betas(&request(fields)), expected); + } + + #[rstest] + #[case::no_betas(&[("x-api-key", "k"), ("anthropic-version", "2023-06-01")], json!({}))] + #[case::blank_beta_header(&[("Anthropic-Beta", " , "), ("x-api-key", "k")], json!({}))] + fn headers_without_any_beta_value_are_untouched( + #[case] input: &[(&str, &str)], + #[case] fields: Value, + ) { + assert_eq!( + with_feature_betas(headers(input), &request(fields)), + headers(input) + ); + } + + #[rstest] + #[case::feature_beta_is_appended( + &[("x-api-key", "k")], + json!({"speed": "fast"}), + &[("x-api-key", "k"), ("anthropic-beta", beta::FAST_MODE_2026_02_01)], + )] + #[case::existing_betas_are_normalized_without_features( + &[("Anthropic-Beta", "web-search-2025-03-05, interleaved-thinking-2025-05-14 ,web-search-2025-03-05"), ("x-api-key", "k")], + json!({}), + &[("x-api-key", "k"), ("anthropic-beta", "interleaved-thinking-2025-05-14,web-search-2025-03-05")], + )] + #[case::existing_advisor_beta_is_kept_without_an_advisor_tool( + &[("anthropic-beta", beta::ADVISOR_TOOL_2026_03_01)], + json!({"tools": []}), + &[("anthropic-beta", beta::ADVISOR_TOOL_2026_03_01)], + )] + #[case::feature_already_sent_is_not_duplicated( + &[("anthropic-beta", beta::FAST_MODE_2026_02_01)], + json!({"speed": "fast"}), + &[("anthropic-beta", beta::FAST_MODE_2026_02_01)], + )] + fn feature_betas_merge_into_the_headers( + #[case] input: &[(&str, &str)], + #[case] fields: Value, + #[case] expected: &[(&str, &str)], + ) { + assert_eq!( + with_feature_betas(headers(input), &request(fields)), + headers(expected) + ); + } + + #[test] + fn differently_cased_beta_header_is_replaced_by_one_sorted_header() { + let merged = with_feature_betas( + headers(&[("Anthropic-Beta", "interleaved-thinking-2025-05-14")]), + &request( + json!({"messages": [{"role": "system", "content": "env", "output_config": {"effort": "low"}}]}), + ), + ); + assert_eq!( + merged, + headers(&[( + "anthropic-beta", + &betas(&[ + "interleaved-thinking-2025-05-14", + beta::PER_TURN_CONTROL_2026_07_01 + ]) + )]) + ); + } + + #[test] + fn every_beta_header_casing_is_unioned_into_one_header() { + let merged = with_feature_betas( + headers(&[ + ("anthropic-beta", "interleaved-thinking-2025-05-14"), + ("Anthropic-Beta", "web-search-2025-03-05"), + ]), + &request(json!({"speed": "fast"})), + ); + assert_eq!( + merged, + headers(&[( + "anthropic-beta", + &betas(&[ + beta::FAST_MODE_2026_02_01, + "interleaved-thinking-2025-05-14", + "web-search-2025-03-05" + ]) + )]) + ); + } + + #[test] + fn unknown_client_betas_survive_alongside_the_added_one() { + let client_betas = [ + "claude-code-20250219", + "interleaved-thinking-2025-05-14", + beta::CONTEXT_MANAGEMENT_2025_06_27, + beta::PER_TURN_CONTROL_2026_07_01, + "effort-2025-11-24", + ]; + let merged = with_feature_betas( + headers(&[("anthropic-beta", &betas(&client_betas))]), + &request( + json!({"messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}]}), + ), + ); + assert_eq!( + merged, + headers(&[( + "anthropic-beta", + &betas(&[ + "claude-code-20250219", + beta::CONTEXT_MANAGEMENT_2025_06_27, + "effort-2025-11-24", + "interleaved-thinking-2025-05-14", + beta::PER_TURN_CONTROL_2026_07_01, + ]) + )]) + ); + } + + #[test] + fn every_feature_merges_with_the_oauth_beta_sorted_and_last() { + let oauth_headers = authenticate_with(&[], Some(OAUTH_TOKEN), &[]).unwrap(); + let all_features = request(json!({ + "compaction": {"enabled": true}, + "output_format": {"type": "json_schema"}, + "speed": "fast", + "tools": [{"type": "advisor_20260301"}, {"type": "tool_search_tool_bm25_20251119"}], + "context_management": {"edits": [{"type": "compact_20260112"}, {"type": "clear_thinking_20251015"}]}, + "messages": [{"role": "user", "content": "hi", "output_config": {"effort": "low"}}], + })); + assert_eq!( + with_feature_betas(oauth_headers, &all_features), + headers(&[ + ("authorization", OAUTH_BEARER), + BROWSER_ACCESS, + ( + "anthropic-beta", + &betas(&[ + beta::ADVANCED_TOOL_USE_2025_11_20, + beta::ADVISOR_TOOL_2026_03_01, + beta::COMPACT_2026_01_12, + beta::COMPACT_2026_09_04, + beta::CONTEXT_MANAGEMENT_2025_06_27, + beta::FAST_MODE_2026_02_01, + ANTHROPIC_OAUTH_BETA_HEADER, + beta::PER_TURN_CONTROL_2026_07_01, + beta::STRUCTURED_OUTPUT, + ]) + ), + ]) + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs index 481d98c4e9d..5adf5fda16f 100644 --- a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/mod.rs @@ -1,2 +1,5 @@ +pub mod handler; +pub mod headers; pub mod streaming_iterator; +pub mod thinking; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/thinking.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/thinking.rs new file mode 100644 index 00000000000..ffa4c8ffeb8 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/thinking.rs @@ -0,0 +1,1182 @@ +use litellm_core_utils::settings::Lookup; +use litellm_types::llms::anthropic_messages::anthropic_request::AnthropicMessagesRequest; +use serde_json::{Map, Value, json}; + +use crate::{ + anthropic::common_utils::AnthropicModelCapabilities, base_llm::chat::transformation::Error, +}; + +pub const ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: u64 = 1024; + +const EFFORT_NAMES: &str = "'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'none'"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct ThinkingBudgets { + pub minimal: u64, + pub low: u64, + pub medium: u64, + pub high: u64, + pub xhigh: u64, + pub max: u64, +} + +impl Default for ThinkingBudgets { + fn default() -> Self { + Self { + minimal: 128, + low: 1024, + medium: 2048, + high: 4096, + xhigh: 8192, + max: 16384, + } + } +} + +impl ThinkingBudgets { + pub fn from_lookup(env: &impl Lookup) -> Self { + let defaults = Self::default(); + let tier = |name: &str, default: u64| { + env.parsed::(&format!("DEFAULT_REASONING_EFFORT_{name}_THINKING_BUDGET")) + .unwrap_or(default) + }; + Self { + minimal: tier("MINIMAL", defaults.minimal), + low: tier("LOW", defaults.low), + medium: tier("MEDIUM", defaults.medium), + high: tier("HIGH", defaults.high), + xhigh: tier("XHIGH", defaults.xhigh), + max: tier("MAX", defaults.max), + } + } + + fn for_effort(&self, reasoning_effort: &str) -> Option { + match reasoning_effort { + "low" => Some(self.low), + "medium" => Some(self.medium), + "high" => Some(self.high), + "xhigh" => Some(self.xhigh), + "max" => Some(self.max), + "minimal" => Some(self.minimal.max(ANTHROPIC_MIN_THINKING_BUDGET_TOKENS)), + _ => None, + } + } + + fn effort_for_budget( + &self, + budget_tokens: u64, + capabilities: &AnthropicModelCapabilities, + ) -> &'static str { + if budget_tokens >= self.xhigh && capabilities.effort_tiers.xhigh { + return "xhigh"; + } + if budget_tokens >= self.high { + return "high"; + } + if budget_tokens >= self.medium { + return "medium"; + } + "low" + } +} + +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct ThinkingContext { + pub capabilities: AnthropicModelCapabilities, + pub budgets: ThinkingBudgets, +} + +fn bad_request(message: String) -> Error { + Error::InvalidRequest(message) +} + +fn thinking_type(thinking: Option<&Value>) -> Option<&str> { + thinking?.get("type")?.as_str() +} + +fn output_config_effort(output_config: Option<&Value>) -> Option<&str> { + output_config?.get("effort")?.as_str() +} + +fn enabled_thinking(budget_tokens: u64) -> Value { + json!({"type": "enabled", "budget_tokens": budget_tokens}) +} + +fn map_reasoning_effort( + reasoning_effort: &str, + context: &ThinkingContext, +) -> Result, Error> { + if reasoning_effort == "none" { + return Ok(None); + } + if context.capabilities.supports_adaptive_thinking { + return Ok(Some(json!({"type": "adaptive", "display": "summarized"}))); + } + context + .budgets + .for_effort(reasoning_effort) + .map(|budget| Some(enabled_thinking(budget))) + .ok_or_else(|| { + bad_request(format!( + "Unmapped reasoning effort: '{reasoning_effort}'. Must be one of: {EFFORT_NAMES}." + )) + }) +} + +fn cap_thinking_budget_to_max_tokens(thinking: Value, max_tokens: Option) -> Option { + let (Some(max_tokens), Some(budget)) = ( + max_tokens, + thinking.get("budget_tokens").and_then(Value::as_u64), + ) else { + return Some(thinking); + }; + if max_tokens <= ANTHROPIC_MIN_THINKING_BUDGET_TOKENS { + return None; + } + if budget < max_tokens { + return Some(thinking); + } + Some(enabled_thinking(max_tokens - 1)) +} + +fn reasoning_effort_to_output_config_effort(reasoning_effort: &str) -> Option<&'static str> { + match reasoning_effort { + "low" | "minimal" => Some("low"), + "medium" => Some("medium"), + "high" => Some("high"), + "xhigh" => Some("xhigh"), + "max" => Some("max"), + _ => None, + } +} + +fn with_default_effort(output_config: Option, effort: &str) -> Value { + let mut config = match output_config { + Some(Value::Object(config)) => config, + _ => Map::new(), + }; + if !config.contains_key("effort") { + config.insert("effort".to_string(), Value::String(effort.to_string())); + } + Value::Object(config) +} + +fn translate_reasoning_effort( + request: AnthropicMessagesRequest, + context: &ThinkingContext, +) -> Result { + let Some(reasoning_effort) = request.reasoning_effort.clone() else { + return Ok(request); + }; + let request = AnthropicMessagesRequest { + reasoning_effort: None, + ..request + }; + let Some(mapped) = map_reasoning_effort(&reasoning_effort, context)? else { + return Ok(AnthropicMessagesRequest { + thinking: None, + output_config: None, + ..request + }); + }; + let Some(fitted) = cap_thinking_budget_to_max_tokens(mapped, request.max_tokens) else { + return Ok(request); + }; + let thinking = Some(request.thinking.clone().unwrap_or(fitted)); + if !context.capabilities.supports_adaptive_thinking { + return Ok(AnthropicMessagesRequest { + thinking, + ..request + }); + } + let effort = reasoning_effort_to_output_config_effort(&reasoning_effort).ok_or_else(|| { + bad_request(format!( + "Invalid reasoning_effort: '{reasoning_effort}'. Must be one of: {EFFORT_NAMES}" + )) + })?; + if let Some(rejection) = context + .capabilities + .effort_level_rejection(effort, &request.model) + { + return Err(bad_request(rejection)); + } + Ok(AnthropicMessagesRequest { + thinking, + output_config: Some(with_default_effort(request.output_config.clone(), effort)), + ..request + }) +} + +fn drop_disabled_thinking( + request: AnthropicMessagesRequest, + context: &ThinkingContext, +) -> AnthropicMessagesRequest { + if !context.capabilities.thinking_always_on + || thinking_type(request.thinking.as_ref()) != Some("disabled") + { + return request; + } + AnthropicMessagesRequest { + thinking: None, + ..request + } +} + +fn translate_legacy_thinking_for_adaptive_model( + request: AnthropicMessagesRequest, + context: &ThinkingContext, +) -> AnthropicMessagesRequest { + let capabilities = &context.capabilities; + if !capabilities.supports_adaptive_thinking + || capabilities.supports_legacy_thinking + || thinking_type(request.thinking.as_ref()) != Some("enabled") + { + return request; + } + let budget = request + .thinking + .as_ref() + .and_then(|thinking| thinking.get("budget_tokens")) + .and_then(Value::as_u64) + .unwrap_or(0); + let effort = context.budgets.effort_for_budget(budget, capabilities); + AnthropicMessagesRequest { + thinking: Some(json!({"type": "adaptive"})), + output_config: Some(with_default_effort(request.output_config.clone(), effort)), + ..request + } +} + +fn output_config_without_effort(output_config: Option) -> Option { + let Some(Value::Object(config)) = output_config else { + return output_config; + }; + if !config.contains_key("effort") { + return Some(Value::Object(config)); + } + let residual: Map = config + .into_iter() + .filter(|(key, _)| key != "effort") + .collect(); + (!residual.is_empty()).then_some(Value::Object(residual)) +} + +fn translate_adaptive_effort_for_non_adaptive_model( + request: AnthropicMessagesRequest, + context: &ThinkingContext, +) -> Result { + let capabilities = &context.capabilities; + if capabilities.supports_adaptive_thinking { + return Ok(request); + } + let effort = output_config_effort(request.output_config.as_ref()).map(str::to_string); + let adaptive_thinking = thinking_type(request.thinking.as_ref()) == Some("adaptive"); + if effort.is_none() && !adaptive_thinking { + return Ok(request); + } + let level_supported = effort.as_deref().is_none_or(|effort| { + capabilities + .effort_level_rejection(effort, &request.model) + .is_none() + }); + if capabilities.supports_effort_param() && (!adaptive_thinking || level_supported) { + return Ok(AnthropicMessagesRequest { + thinking: if adaptive_thinking { + None + } else { + request.thinking.clone() + }, + ..request + }); + } + let legacy = if capabilities.supports_reasoning { + map_reasoning_effort( + effort + .as_deref() + .filter(|effort| !effort.is_empty()) + .unwrap_or("medium"), + context, + )? + } else { + None + }; + let capped = + legacy.and_then(|thinking| cap_thinking_budget_to_max_tokens(thinking, request.max_tokens)); + Ok(AnthropicMessagesRequest { + thinking: capped, + output_config: output_config_without_effort(request.output_config.clone()), + ..request + }) +} + +fn drop_incompatible_temperature_for_thinking( + request: AnthropicMessagesRequest, + context: &ThinkingContext, +) -> AnthropicMessagesRequest { + if context.capabilities.supports_adaptive_thinking { + return request; + } + let pinned = request + .temperature + .is_some_and(|temperature| temperature != 1.0); + let thinking_enabled = thinking_type(request.thinking.as_ref()) == Some("enabled"); + let effort_enabled = output_config_effort(request.output_config.as_ref()).is_some(); + if !pinned || !(thinking_enabled || effort_enabled) { + return request; + } + AnthropicMessagesRequest { + temperature: None, + ..request + } +} + +pub fn translate_thinking( + request: AnthropicMessagesRequest, + context: &ThinkingContext, +) -> Result { + let request = translate_reasoning_effort(request, context)?; + let request = drop_disabled_thinking(request, context); + let request = translate_legacy_thinking_for_adaptive_model(request, context); + let request = translate_adaptive_effort_for_non_adaptive_model(request, context)?; + Ok(drop_incompatible_temperature_for_thinking(request, context)) +} + +#[cfg(test)] +mod tests { + use rstest::{fixture, rstest}; + + use super::*; + use crate::anthropic::common_utils::SupportedEffortTiers; + + const EFFORT_CHOICES: &str = "'minimal', 'low', 'medium', 'high', 'xhigh', 'max', 'none'"; + + fn request(fields: Value) -> AnthropicMessagesRequest { + let mut body = + json!({"model": "claude", "messages": [{"role": "user", "content": "Hello"}]}); + body.as_object_mut() + .unwrap() + .extend(fields.as_object().unwrap().clone()); + serde_json::from_value(body).unwrap() + } + + fn context(capabilities: AnthropicModelCapabilities) -> ThinkingContext { + ThinkingContext { + capabilities, + budgets: ThinkingBudgets::default(), + } + } + + fn translate( + capabilities: AnthropicModelCapabilities, + fields: Value, + ) -> Result { + translate_thinking(request(fields), &context(capabilities)) + } + + fn overridden_budgets(overrides: &[(&str, &str)]) -> ThinkingBudgets { + let env = |name: &str| { + overrides + .iter() + .find(|(tier, _)| { + name == format!("DEFAULT_REASONING_EFFORT_{tier}_THINKING_BUDGET") + }) + .map(|(_, value)| value.to_string()) + }; + ThinkingBudgets::from_lookup(&env) + } + + fn claude_code_payload(effort: &str, max_tokens: u64) -> Value { + json!({"max_tokens": max_tokens, "thinking": {"type": "adaptive"}, "output_config": {"effort": effort}}) + } + + fn with_temperature(fields: Value, temperature: f64) -> Value { + let mut fields = fields; + fields + .as_object_mut() + .unwrap() + .insert("temperature".to_string(), json!(temperature)); + fields + } + + #[fixture] + fn haiku_3_5() -> AnthropicModelCapabilities { + AnthropicModelCapabilities::default() + } + + #[fixture] + fn haiku_4_5() -> AnthropicModelCapabilities { + AnthropicModelCapabilities { + supports_reasoning: true, + ..Default::default() + } + } + + #[fixture] + fn opus_4_5() -> AnthropicModelCapabilities { + AnthropicModelCapabilities { + supports_reasoning: true, + supports_output_config: true, + ..Default::default() + } + } + + #[fixture] + fn sonnet_4_6() -> AnthropicModelCapabilities { + AnthropicModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_legacy_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + max: true, + ..Default::default() + }, + ..Default::default() + } + } + + #[fixture] + fn opus_4_7() -> AnthropicModelCapabilities { + AnthropicModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_output_config: true, + effort_tiers: SupportedEffortTiers { + xhigh: true, + max: true, + ..Default::default() + }, + ..Default::default() + } + } + + #[fixture] + fn fable_5_1() -> AnthropicModelCapabilities { + AnthropicModelCapabilities { + thinking_always_on: true, + ..opus_4_7() + } + } + + #[fixture] + fn newfamily_6() -> AnthropicModelCapabilities { + AnthropicModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + ..Default::default() + } + } + + #[rstest] + #[case::minimal_maps_to_low(opus_4_7(), "minimal", "low")] + #[case::low(opus_4_7(), "low", "low")] + #[case::medium(opus_4_7(), "medium", "medium")] + #[case::high(opus_4_7(), "high", "high")] + #[case::xhigh_with_xhigh_tier(opus_4_7(), "xhigh", "xhigh")] + #[case::max(opus_4_7(), "max", "max")] + #[case::minimal_maps_to_low_on_4_6(sonnet_4_6(), "minimal", "low")] + #[case::low_on_4_6(sonnet_4_6(), "low", "low")] + #[case::max_without_max_tier_is_allowed_on_adaptive_models(newfamily_6(), "max", "max")] + fn reasoning_effort_on_adaptive_model_becomes_summarized_adaptive_thinking_and_effort( + #[case] capabilities: AnthropicModelCapabilities, + #[case] reasoning_effort: &str, + #[case] expected_effort: &str, + ) { + assert_eq!( + translate( + capabilities, + json!({"max_tokens": 1024, "reasoning_effort": reasoning_effort}) + ), + Ok(request(json!({ + "max_tokens": 1024, + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"effort": expected_effort} + }))) + ); + } + + #[rstest] + #[case::adaptive_shape_is_not_dropped_for_small_max_tokens( + opus_4_7(), + json!({"max_tokens": 64, "reasoning_effort": "high"}), + json!({"max_tokens": 64, "thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "high"}}) + )] + #[case::caller_output_config_effort_wins( + opus_4_7(), + json!({"max_tokens": 1024, "reasoning_effort": "low", "output_config": {"effort": "max"}}), + json!({"max_tokens": 1024, "thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "max"}}) + )] + #[case::effort_merges_into_caller_output_config( + opus_4_7(), + json!({"max_tokens": 1024, "reasoning_effort": "high", "output_config": {"format": {"type": "json_schema"}}}), + json!({ + "max_tokens": 1024, + "thinking": {"type": "adaptive", "display": "summarized"}, + "output_config": {"format": {"type": "json_schema"}, "effort": "high"} + }) + )] + #[case::non_object_output_config_is_replaced( + opus_4_7(), + json!({"max_tokens": 1024, "reasoning_effort": "high", "output_config": "bogus"}), + json!({"max_tokens": 1024, "thinking": {"type": "adaptive", "display": "summarized"}, "output_config": {"effort": "high"}}) + )] + #[case::caller_thinking_and_output_config_win( + sonnet_4_6(), + json!({ + "max_tokens": 16000, + "reasoning_effort": "low", + "thinking": {"type": "enabled", "budget_tokens": 8000}, + "output_config": {"effort": "high"} + }), + json!({ + "max_tokens": 16000, + "thinking": {"type": "enabled", "budget_tokens": 8000}, + "output_config": {"effort": "high"} + }) + )] + #[case::caller_legacy_thinking_is_then_translated_while_reasoning_effort_level_stays( + opus_4_7(), + json!({"max_tokens": 16000, "reasoning_effort": "low", "thinking": {"type": "enabled", "budget_tokens": 8000}}), + json!({"max_tokens": 16000, "thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}) + )] + #[case::caller_disabled_thinking_is_kept_then_omitted_on_always_on_model( + fable_5_1(), + json!({"max_tokens": 1024, "reasoning_effort": "high", "thinking": {"type": "disabled"}}), + json!({"max_tokens": 1024, "output_config": {"effort": "high"}}) + )] + #[case::non_adaptive_model_gets_no_output_config( + opus_4_5(), + json!({"max_tokens": 8192, "reasoning_effort": "high"}), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + #[case::caller_thinking_wins_on_non_adaptive_model( + opus_4_5(), + json!({"max_tokens": 16000, "reasoning_effort": "low", "thinking": {"type": "enabled", "budget_tokens": 8000}}), + json!({"max_tokens": 16000, "thinking": {"type": "enabled", "budget_tokens": 8000}}) + )] + #[case::caller_thinking_survives_when_mapped_budget_cannot_fit( + opus_4_5(), + json!({"max_tokens": 1024, "reasoning_effort": "low", "thinking": {"type": "enabled", "budget_tokens": 8000}}), + json!({"max_tokens": 1024, "thinking": {"type": "enabled", "budget_tokens": 8000}}) + )] + #[case::missing_max_tokens_leaves_budget_uncapped( + haiku_4_5(), + json!({"reasoning_effort": "high"}), + json!({"thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + #[case::budget_below_max_tokens_is_kept( + haiku_4_5(), + json!({"max_tokens": 4097, "reasoning_effort": "high"}), + json!({"max_tokens": 4097, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + #[case::budget_equal_to_max_tokens_is_capped( + haiku_4_5(), + json!({"max_tokens": 4096, "reasoning_effort": "high"}), + json!({"max_tokens": 4096, "thinking": {"type": "enabled", "budget_tokens": 4095}}) + )] + #[case::budget_above_max_tokens_is_capped( + haiku_4_5(), + json!({"max_tokens": 4000, "reasoning_effort": "xhigh"}), + json!({"max_tokens": 4000, "thinking": {"type": "enabled", "budget_tokens": 3999}}) + )] + #[case::max_tokens_just_above_min_budget_caps_to_min_budget( + haiku_4_5(), + json!({"max_tokens": 1025, "reasoning_effort": "xhigh"}), + json!({"max_tokens": 1025, "thinking": {"type": "enabled", "budget_tokens": 1024}}) + )] + #[case::max_tokens_at_min_budget_drops_thinking( + haiku_4_5(), + json!({"max_tokens": 1024, "reasoning_effort": "xhigh"}), + json!({"max_tokens": 1024}) + )] + #[case::pinned_temperature_is_dropped_after_thinking_is_synthesized( + haiku_4_5(), + json!({"max_tokens": 8192, "reasoning_effort": "low", "temperature": 0}), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 1024}}) + )] + fn reasoning_effort_is_translated( + #[case] capabilities: AnthropicModelCapabilities, + #[case] input: Value, + #[case] expected: Value, + ) { + assert_eq!(translate(capabilities, input), Ok(request(expected))); + } + + #[rstest] + #[case::minimal_floors_at_min_budget("minimal", 1024)] + #[case::low("low", 1024)] + #[case::medium("medium", 2048)] + #[case::high("high", 4096)] + #[case::xhigh("xhigh", 8192)] + #[case::max("max", 16384)] + fn reasoning_effort_on_non_adaptive_model_uses_the_tier_budget( + haiku_4_5: AnthropicModelCapabilities, + #[case] reasoning_effort: &str, + #[case] expected_budget: u64, + ) { + assert_eq!( + translate( + haiku_4_5, + json!({"max_tokens": 32000, "reasoning_effort": reasoning_effort}) + ), + Ok(request(json!({ + "max_tokens": 32000, + "thinking": {"type": "enabled", "budget_tokens": expected_budget} + }))) + ); + } + + #[rstest] + #[case::adaptive_model(opus_4_7())] + #[case::effort_capable_model(opus_4_5())] + #[case::budget_model(haiku_4_5())] + fn reasoning_effort_none_clears_thinking_and_output_config( + #[case] capabilities: AnthropicModelCapabilities, + ) { + assert_eq!( + translate( + capabilities, + json!({ + "max_tokens": 1024, + "reasoning_effort": "none", + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "high"} + }) + ), + Ok(request(json!({"max_tokens": 1024}))) + ); + } + + #[rstest] + #[case::bogus_on_budget_model( + opus_4_5(), + json!({"max_tokens": 1024, "reasoning_effort": "bogus"}), + format!("Unmapped reasoning effort: 'bogus'. Must be one of: {EFFORT_CHOICES}.") + )] + #[case::disabled_on_budget_model( + haiku_4_5(), + json!({"max_tokens": 1024, "reasoning_effort": "disabled"}), + format!("Unmapped reasoning effort: 'disabled'. Must be one of: {EFFORT_CHOICES}.") + )] + #[case::empty_on_budget_model( + haiku_4_5(), + json!({"max_tokens": 1024, "reasoning_effort": ""}), + format!("Unmapped reasoning effort: ''. Must be one of: {EFFORT_CHOICES}.") + )] + #[case::invalid_on_adaptive_model( + opus_4_7(), + json!({"max_tokens": 1024, "reasoning_effort": "invalid"}), + format!("Invalid reasoning_effort: 'invalid'. Must be one of: {EFFORT_CHOICES}") + )] + #[case::disabled_on_adaptive_model( + opus_4_7(), + json!({"max_tokens": 1024, "reasoning_effort": "disabled"}), + format!("Invalid reasoning_effort: 'disabled'. Must be one of: {EFFORT_CHOICES}") + )] + #[case::empty_on_adaptive_model( + opus_4_7(), + json!({"max_tokens": 1024, "reasoning_effort": ""}), + format!("Invalid reasoning_effort: ''. Must be one of: {EFFORT_CHOICES}") + )] + #[case::xhigh_without_xhigh_tier_on_4_6( + sonnet_4_6(), + json!({"max_tokens": 1024, "reasoning_effort": "xhigh"}), + "effort='xhigh' is not supported by this model. Got model: claude".to_string() + )] + #[case::xhigh_without_xhigh_tier_on_unmapped_adaptive_model( + newfamily_6(), + json!({"max_tokens": 1024, "reasoning_effort": "xhigh"}), + "effort='xhigh' is not supported by this model. Got model: claude".to_string() + )] + #[case::unrecognized_adaptive_effort_on_budget_model( + haiku_4_5(), + claude_code_payload("turbo", 8192), + format!("Unmapped reasoning effort: 'turbo'. Must be one of: {EFFORT_CHOICES}.") + )] + fn unsupported_effort_is_a_request_error( + #[case] capabilities: AnthropicModelCapabilities, + #[case] input: Value, + #[case] expected_message: String, + ) { + assert_eq!( + translate(capabilities, input), + Err(Error::InvalidRequest(expected_message)) + ); + } + + #[rstest] + #[case::omitted_on_always_on_model(fable_5_1(), json!({"type": "disabled"}), None)] + #[case::kept_on_adaptive_model(opus_4_7(), json!({"type": "disabled"}), Some(json!({"type": "disabled"})))] + #[case::kept_on_budget_model(haiku_4_5(), json!({"type": "disabled"}), Some(json!({"type": "disabled"})))] + #[case::adaptive_kept_on_always_on_model( + fable_5_1(), + json!({"type": "adaptive"}), + Some(json!({"type": "adaptive"})) + )] + fn disabled_thinking_is_omitted_only_for_always_on_models( + #[case] capabilities: AnthropicModelCapabilities, + #[case] thinking: Value, + #[case] expected_thinking: Option, + ) { + let expected = match expected_thinking { + Some(thinking) => json!({"max_tokens": 64, "thinking": thinking}), + None => json!({"max_tokens": 64}), + }; + assert_eq!( + translate( + capabilities, + json!({"max_tokens": 64, "thinking": thinking}) + ), + Ok(request(expected)) + ); + } + + #[rstest] + #[case::far_above_xhigh_budget(opus_4_7(), json!(16384), "xhigh")] + #[case::at_xhigh_budget(opus_4_7(), json!(8192), "xhigh")] + #[case::below_xhigh_budget(opus_4_7(), json!(8191), "high")] + #[case::xhigh_budget_without_xhigh_tier(newfamily_6(), json!(8192), "high")] + #[case::large_budget_without_xhigh_tier(newfamily_6(), json!(31999), "high")] + #[case::at_high_budget(opus_4_7(), json!(4096), "high")] + #[case::below_high_budget(opus_4_7(), json!(4095), "medium")] + #[case::at_medium_budget(opus_4_7(), json!(2048), "medium")] + #[case::below_medium_budget(opus_4_7(), json!(2047), "low")] + #[case::tiny_budget(opus_4_7(), json!(1), "low")] + #[case::missing_budget(opus_4_7(), Value::Null, "low")] + #[case::always_on_model(fable_5_1(), json!(24000), "xhigh")] + fn legacy_thinking_is_bucketed_into_adaptive_effort_on_adaptive_only_models( + #[case] capabilities: AnthropicModelCapabilities, + #[case] budget_tokens: Value, + #[case] expected_effort: &str, + ) { + let thinking = match budget_tokens { + Value::Null => json!({"type": "enabled"}), + budget_tokens => json!({"type": "enabled", "budget_tokens": budget_tokens}), + }; + assert_eq!( + translate( + capabilities, + json!({"max_tokens": 1024, "thinking": thinking}) + ), + Ok(request(json!({ + "max_tokens": 1024, + "thinking": {"type": "adaptive"}, + "output_config": {"effort": expected_effort} + }))) + ); + } + + #[rstest] + #[case::verbatim_on_model_accepting_legacy_thinking( + sonnet_4_6(), + json!({"max_tokens": 1024, "thinking": {"type": "enabled", "budget_tokens": 31999}}), + json!({"max_tokens": 1024, "thinking": {"type": "enabled", "budget_tokens": 31999}}) + )] + #[case::verbatim_with_explicit_output_config_on_model_accepting_legacy_thinking( + sonnet_4_6(), + json!({"max_tokens": 1024, "thinking": {"type": "enabled", "budget_tokens": 31999}, "output_config": {"effort": "low"}}), + json!({"max_tokens": 1024, "thinking": {"type": "enabled", "budget_tokens": 31999}, "output_config": {"effort": "low"}}) + )] + #[case::verbatim_on_non_adaptive_model( + opus_4_5(), + json!({"max_tokens": 1024, "thinking": {"type": "enabled", "budget_tokens": 31999}}), + json!({"max_tokens": 1024, "thinking": {"type": "enabled", "budget_tokens": 31999}}) + )] + #[case::caller_output_config_effort_wins( + opus_4_7(), + json!({ + "max_tokens": 32000, + "thinking": {"type": "enabled", "budget_tokens": 31999}, + "output_config": {"effort": "low", "format": {"type": "json_schema"}} + }), + json!({ + "max_tokens": 32000, + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "low", "format": {"type": "json_schema"}} + }) + )] + #[case::effort_merges_into_caller_output_config( + opus_4_7(), + json!({ + "max_tokens": 32000, + "thinking": {"type": "enabled", "budget_tokens": 4096}, + "output_config": {"format": {"type": "json_schema"}} + }), + json!({ + "max_tokens": 32000, + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "high", "format": {"type": "json_schema"}} + }) + )] + #[case::adaptive_thinking_is_left_alone( + opus_4_7(), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive", "display": "summarized"}}), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive", "display": "summarized"}}) + )] + fn legacy_thinking_on_adaptive_capable_models( + #[case] capabilities: AnthropicModelCapabilities, + #[case] input: Value, + #[case] expected: Value, + ) { + assert_eq!(translate(capabilities, input), Ok(request(expected))); + } + + #[rstest] + #[case::bare_adaptive_becomes_medium_budget_on_budget_model( + haiku_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}}) + )] + #[case::medium_effort_becomes_medium_budget_on_budget_model( + haiku_4_5(), + claude_code_payload("medium", 8192), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}}) + )] + #[case::empty_effort_becomes_medium_budget_on_budget_model( + haiku_4_5(), + claude_code_payload("", 8192), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}}) + )] + #[case::high_effort_becomes_high_budget_on_budget_model( + haiku_4_5(), + claude_code_payload("high", 8192), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + #[case::effort_only_becomes_budget_on_budget_model( + haiku_4_5(), + json!({"max_tokens": 8192, "output_config": {"effort": "high"}}), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + #[case::effort_replaces_caller_legacy_budget_on_budget_model( + haiku_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 3000}, "output_config": {"effort": "high"}}), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + #[case::residual_output_config_survives_effort_translation( + haiku_4_5(), + json!({ + "max_tokens": 8192, + "thinking": {"type": "adaptive"}, + "output_config": {"effort": "medium", "format": {"type": "json_schema"}} + }), + json!({ + "max_tokens": 8192, + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "output_config": {"format": {"type": "json_schema"}} + }) + )] + #[case::effortless_output_config_is_kept( + haiku_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}, "output_config": {"format": {"type": "json_schema"}}}), + json!({ + "max_tokens": 8192, + "thinking": {"type": "enabled", "budget_tokens": 2048}, + "output_config": {"format": {"type": "json_schema"}} + }) + )] + #[case::empty_output_config_is_kept( + haiku_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}, "output_config": {}}), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}, "output_config": {}}) + )] + #[case::missing_max_tokens_leaves_budget_uncapped( + haiku_4_5(), + json!({"thinking": {"type": "adaptive"}}), + json!({"thinking": {"type": "enabled", "budget_tokens": 2048}}) + )] + #[case::budget_is_capped_below_max_tokens( + haiku_4_5(), + claude_code_payload("high", 3000), + json!({"max_tokens": 3000, "thinking": {"type": "enabled", "budget_tokens": 2999}}) + )] + #[case::max_tokens_just_above_min_budget_caps_to_min_budget( + haiku_4_5(), + claude_code_payload("medium", 1025), + json!({"max_tokens": 1025, "thinking": {"type": "enabled", "budget_tokens": 1024}}) + )] + #[case::max_tokens_at_min_budget_drops_thinking_and_effort( + haiku_4_5(), + claude_code_payload("medium", 1024), + json!({"max_tokens": 1024}) + )] + #[case::max_tokens_below_min_budget_drops_thinking_and_effort( + haiku_4_5(), + claude_code_payload("medium", 512), + json!({"max_tokens": 512}) + )] + #[case::bare_adaptive_is_dropped_on_non_reasoning_model( + haiku_3_5(), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}), + json!({"max_tokens": 8192}) + )] + #[case::adaptive_and_effort_are_dropped_on_non_reasoning_model( + haiku_3_5(), + claude_code_payload("medium", 8192), + json!({"max_tokens": 8192}) + )] + #[case::effort_only_is_dropped_on_non_reasoning_model( + haiku_3_5(), + json!({"max_tokens": 8192, "output_config": {"effort": "high", "format": {"type": "json_schema"}}}), + json!({"max_tokens": 8192, "output_config": {"format": {"type": "json_schema"}}}) + )] + #[case::supported_effort_is_kept_and_adaptive_thinking_dropped_on_effort_model( + opus_4_5(), + claude_code_payload("medium", 8192), + json!({"max_tokens": 8192, "output_config": {"effort": "medium"}}) + )] + #[case::bare_adaptive_is_dropped_on_effort_model( + opus_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}), + json!({"max_tokens": 8192}) + )] + #[case::effort_only_is_left_alone_on_effort_model( + opus_4_5(), + json!({"max_tokens": 8192, "output_config": {"effort": "high"}}), + json!({"max_tokens": 8192, "output_config": {"effort": "high"}}) + )] + #[case::unsupported_effort_only_is_left_for_provider_normalization( + opus_4_5(), + json!({"max_tokens": 4096, "output_config": {"effort": "xhigh"}}), + json!({"max_tokens": 4096, "output_config": {"effort": "xhigh"}}) + )] + #[case::legacy_thinking_is_kept_beside_native_effort_on_effort_model( + opus_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}, "output_config": {"effort": "high"}}), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}, "output_config": {"effort": "high"}}) + )] + #[case::unsupported_xhigh_with_adaptive_thinking_falls_back_to_budget( + opus_4_5(), + claude_code_payload("xhigh", 64000), + json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": 8192}}) + )] + #[case::unsupported_max_with_adaptive_thinking_falls_back_to_budget( + opus_4_5(), + claude_code_payload("max", 64000), + json!({"max_tokens": 64000, "thinking": {"type": "enabled", "budget_tokens": 16384}}) + )] + #[case::bare_adaptive_is_native_on_4_6( + sonnet_4_6(), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}) + )] + #[case::adaptive_payload_is_native_on_4_6( + sonnet_4_6(), + claude_code_payload("high", 8192), + claude_code_payload("high", 8192) + )] + #[case::request_without_adaptive_interface_is_left_alone( + haiku_4_5(), + json!({"max_tokens": 1024}), + json!({"max_tokens": 1024}) + )] + fn adaptive_interface_is_reshaped_for_non_adaptive_models( + #[case] capabilities: AnthropicModelCapabilities, + #[case] input: Value, + #[case] expected: Value, + ) { + assert_eq!(translate(capabilities, input), Ok(request(expected))); + } + + #[rstest] + #[case::adaptive_downgraded_to_enabled_thinking( + haiku_4_5(), + claude_code_payload("medium", 8192), + 0.0, + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}}) + )] + #[case::bare_adaptive_downgraded_to_enabled_thinking( + haiku_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "adaptive"}}), + 0.0, + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}}) + )] + #[case::reasoning_effort_synthesized_enabled_thinking( + haiku_4_5(), + json!({"max_tokens": 8192, "reasoning_effort": "high"}), + 0.2, + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + #[case::above_one_with_enabled_thinking( + haiku_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}}), + 1.5, + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}}) + )] + #[case::native_effort_kept_on_effort_model( + opus_4_5(), + claude_code_payload("medium", 8192), + 0.0, + json!({"max_tokens": 8192, "output_config": {"effort": "medium"}}) + )] + #[case::effort_only_on_effort_model( + opus_4_5(), + json!({"max_tokens": 8192, "output_config": {"effort": "high"}}), + 0.0, + json!({"max_tokens": 8192, "output_config": {"effort": "high"}}) + )] + fn pinned_temperature_is_dropped_when_thinking_or_effort_survives_on_non_adaptive_model( + #[case] capabilities: AnthropicModelCapabilities, + #[case] input: Value, + #[case] temperature: f64, + #[case] expected: Value, + ) { + assert_eq!( + translate(capabilities, with_temperature(input, temperature)), + Ok(request(expected)) + ); + } + + #[rstest] + #[case::temperature_one_with_enabled_thinking( + haiku_4_5(), + claude_code_payload("medium", 8192), + 1.0, + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 2048}}) + )] + #[case::thinking_dropped_for_small_max_tokens( + haiku_4_5(), + claude_code_payload("medium", 512), + 0.0, + json!({"max_tokens": 512}) + )] + #[case::thinking_dropped_on_non_reasoning_model( + haiku_3_5(), + claude_code_payload("medium", 8192), + 0.0, + json!({"max_tokens": 8192}) + )] + #[case::disabled_thinking( + haiku_4_5(), + json!({"max_tokens": 8192, "thinking": {"type": "disabled"}}), + 0.0, + json!({"max_tokens": 8192, "thinking": {"type": "disabled"}}) + )] + #[case::no_thinking(haiku_4_5(), json!({"max_tokens": 8192}), 0.0, json!({"max_tokens": 8192}))] + #[case::output_config_without_effort( + haiku_4_5(), + json!({"max_tokens": 8192, "output_config": {"format": {"type": "json_schema"}}}), + 0.0, + json!({"max_tokens": 8192, "output_config": {"format": {"type": "json_schema"}}}) + )] + #[case::adaptive_model( + opus_4_7(), + claude_code_payload("medium", 8192), + 0.0, + claude_code_payload("medium", 8192) + )] + #[case::legacy_thinking_on_adaptive_model( + sonnet_4_6(), + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}), + 0.0, + json!({"max_tokens": 8192, "thinking": {"type": "enabled", "budget_tokens": 4096}}) + )] + fn temperature_is_kept( + #[case] capabilities: AnthropicModelCapabilities, + #[case] input: Value, + #[case] temperature: f64, + #[case] expected: Value, + ) { + assert_eq!( + translate(capabilities, with_temperature(input, temperature)), + Ok(request(with_temperature(expected, temperature))) + ); + } + + #[rstest] + #[case::minimal("MINIMAL", ThinkingBudgets { minimal: 5000, ..ThinkingBudgets::default() })] + #[case::low("LOW", ThinkingBudgets { low: 5000, ..ThinkingBudgets::default() })] + #[case::medium("MEDIUM", ThinkingBudgets { medium: 5000, ..ThinkingBudgets::default() })] + #[case::high("HIGH", ThinkingBudgets { high: 5000, ..ThinkingBudgets::default() })] + #[case::xhigh("XHIGH", ThinkingBudgets { xhigh: 5000, ..ThinkingBudgets::default() })] + #[case::max("MAX", ThinkingBudgets { max: 5000, ..ThinkingBudgets::default() })] + fn each_tier_budget_reads_only_its_own_environment_override( + #[case] tier: &str, + #[case] expected: ThinkingBudgets, + ) { + assert_eq!(overridden_budgets(&[(tier, "5000")]), expected); + } + + #[rstest] + #[case::whitespace_is_trimmed(" 6000 ", 6000)] + #[case::unparseable_value_keeps_default("lots", 4096)] + fn environment_override_parsing(#[case] raw: &str, #[case] expected_high: u64) { + assert_eq!( + overridden_budgets(&[("HIGH", raw)]), + ThinkingBudgets { + high: expected_high, + ..ThinkingBudgets::default() + } + ); + } + + #[rstest] + #[case::reasoning_effort_uses_overridden_budget( + &[("HIGH", "6000")], + haiku_4_5(), + json!({"max_tokens": 32000, "reasoning_effort": "high"}), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 6000}}) + )] + #[case::minimal_override_below_min_budget_is_floored( + &[("MINIMAL", "512")], + haiku_4_5(), + json!({"max_tokens": 32000, "reasoning_effort": "minimal"}), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 1024}}) + )] + #[case::minimal_override_above_min_budget_is_used( + &[("MINIMAL", "2000")], + haiku_4_5(), + json!({"max_tokens": 32000, "reasoning_effort": "minimal"}), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 2000}}) + )] + #[case::adaptive_fallback_uses_overridden_medium_budget( + &[("MEDIUM", "3000")], + haiku_4_5(), + json!({"max_tokens": 32000, "thinking": {"type": "adaptive"}}), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 3000}}) + )] + #[case::legacy_bucket_below_overridden_high_budget( + &[("HIGH", "6000")], + opus_4_7(), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 5999}}), + json!({"max_tokens": 32000, "thinking": {"type": "adaptive"}, "output_config": {"effort": "medium"}}) + )] + #[case::legacy_bucket_at_overridden_high_budget( + &[("HIGH", "6000")], + opus_4_7(), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 6000}}), + json!({"max_tokens": 32000, "thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}) + )] + #[case::legacy_bucket_below_overridden_xhigh_budget( + &[("XHIGH", "20000")], + opus_4_7(), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 19999}}), + json!({"max_tokens": 32000, "thinking": {"type": "adaptive"}, "output_config": {"effort": "high"}}) + )] + #[case::legacy_bucket_at_overridden_medium_budget( + &[("MEDIUM", "3000")], + opus_4_7(), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 3000}}), + json!({"max_tokens": 32000, "thinking": {"type": "adaptive"}, "output_config": {"effort": "medium"}}) + )] + #[case::legacy_bucket_below_overridden_medium_budget( + &[("MEDIUM", "3000")], + opus_4_7(), + json!({"max_tokens": 32000, "thinking": {"type": "enabled", "budget_tokens": 2999}}), + json!({"max_tokens": 32000, "thinking": {"type": "adaptive"}, "output_config": {"effort": "low"}}) + )] + fn translation_honors_budget_overrides( + #[case] overrides: &[(&str, &str)], + #[case] capabilities: AnthropicModelCapabilities, + #[case] input: Value, + #[case] expected: Value, + ) { + let context = ThinkingContext { + capabilities, + budgets: overridden_budgets(overrides), + }; + assert_eq!( + translate_thinking(request(input), &context), + Ok(request(expected)) + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs index c791749ac6d..59280c04a70 100644 --- a/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/experimental_pass_through/messages/transformation.rs @@ -1,9 +1,28 @@ -use crate::base_llm::{ - anthropic_messages::transformation::BaseAnthropicMessagesConfig, chat::transformation::Error, +use litellm_core_utils::settings::{Lookup, ProcessEnvironment}; +use litellm_types::llms::anthropic_messages::anthropic_request::AnthropicMessagesRequest; +use serde_json::{Map, Value, json}; + +use super::{ + headers::{authenticate, with_feature_betas}, + thinking::{ThinkingBudgets, ThinkingContext, translate_thinking}, +}; +use crate::{ + anthropic::common_utils::{ + AnthropicModelCapabilities, has_advisor_tool, strip_advisor_blocks, + strip_encrypted_reasoning_blocks, + }, + base_llm::{ + anthropic_messages::transformation::{ + BaseAnthropicMessagesConfig, Headers, MessagesTransformContext, + }, + chat::transformation::Error, + }, }; const ANTHROPIC_API_KEY_ENV: &str = "ANTHROPIC_API_KEY"; +const ANTHROPIC_AUTH_TOKEN_ENV: &str = "ANTHROPIC_AUTH_TOKEN"; const ANTHROPIC_API_BASE_ENV: &str = "ANTHROPIC_API_BASE"; +const ANTHROPIC_BASE_URL_ENV: &str = "ANTHROPIC_BASE_URL"; const DEFAULT_ANTHROPIC_API_BASE: &str = "https://api.anthropic.com"; const MESSAGES_PATH_SUFFIX: &str = "/v1/messages"; @@ -11,6 +30,26 @@ pub struct AnthropicMessagesConfig; pub const ANTHROPIC_MESSAGES_CONFIG: AnthropicMessagesConfig = AnthropicMessagesConfig; +impl MessagesTransformContext { + pub fn new(capabilities: AnthropicModelCapabilities, drop_params: bool) -> Self { + Self::with_lookup(capabilities, drop_params, &ProcessEnvironment) + } + + pub fn with_lookup( + capabilities: AnthropicModelCapabilities, + drop_params: bool, + env: &impl Lookup, + ) -> Self { + Self { + thinking: ThinkingContext { + capabilities, + budgets: ThinkingBudgets::from_lookup(env), + }, + drop_params, + } + } +} + impl BaseAnthropicMessagesConfig for AnthropicMessagesConfig { fn get_complete_url( &self, @@ -21,6 +60,35 @@ impl BaseAnthropicMessagesConfig for AnthropicMessagesConfig { Ok(complete_anthropic_url(api_base, env_lookup)) } + fn transform_anthropic_messages_request( + &self, + request: AnthropicMessagesRequest, + context: &MessagesTransformContext, + ) -> Result { + if request.max_tokens.is_none() { + return Err(Error::InvalidRequest( + "max_tokens is required for Anthropic /v1/messages API".to_string(), + )); + } + let request = drop_unsupported_params(request, context)?; + let request = translate_thinking(request, &context.thinking)?; + let context_management = request + .context_management + .as_ref() + .and_then(map_openai_context_management_to_anthropic) + .or_else(|| request.context_management.clone()); + let messages = if has_advisor_tool(request.tools.as_deref()) { + request.messages + } else { + strip_advisor_blocks(request.messages) + }; + Ok(AnthropicMessagesRequest { + messages: strip_encrypted_reasoning_blocks(messages), + context_management, + ..request + }) + } + fn resolve_api_key( &self, api_key: Option<&str>, @@ -28,6 +96,113 @@ impl BaseAnthropicMessagesConfig for AnthropicMessagesConfig { ) -> Result { resolve_anthropic_api_key(api_key, env_lookup).map_err(Error::from) } + + fn secret_names(&self) -> &'static [&'static str] { + &[ + ANTHROPIC_API_KEY_ENV, + ANTHROPIC_AUTH_TOKEN_ENV, + ANTHROPIC_API_BASE_ENV, + ANTHROPIC_BASE_URL_ENV, + ] + } + + fn authenticate( + &self, + headers: Headers, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + authenticate(headers, api_key, env_lookup).map_err(Error::from) + } + + fn request_headers(&self, headers: Headers, request: &AnthropicMessagesRequest) -> Headers { + with_feature_betas(headers, request) + } +} + +fn unsupported_param(model: &str, param: &str, value: &str, hint: &str) -> Error { + Error::InvalidRequest(format!( + "{model} does not support {param}={value}. {hint}To drop unsupported params, set `litellm.drop_params = True`." + )) +} + +fn drop_unsupported_params( + request: AnthropicMessagesRequest, + context: &MessagesTransformContext, +) -> Result { + let capabilities = &context.thinking.capabilities; + let model = request.model.clone(); + let reject = |param: &str, value: String, hint: &str| -> Result<(), Error> { + if context.drop_params { + return Ok(()); + } + Err(unsupported_param(&model, param, &value, hint)) + }; + let speed = match request.speed.as_deref() { + Some(speed) if !capabilities.supports_speed => { + reject("speed", format!("'{speed}'"), "")?; + None + } + _ => request.speed.clone(), + }; + if capabilities.supports_sampling_params { + return Ok(AnthropicMessagesRequest { speed, ..request }); + } + let temperature = match request.temperature { + Some(temperature) if temperature != 1.0 => { + reject( + "temperature", + json!(temperature).to_string(), + "Only temperature=1 is supported. ", + )?; + None + } + temperature => temperature, + }; + if let Some(top_p) = request.top_p { + reject("top_p", json!(top_p).to_string(), "")?; + } + if let Some(top_k) = request.top_k { + reject("top_k", json!(top_k).to_string(), "")?; + } + Ok(AnthropicMessagesRequest { + speed, + temperature, + top_p: None, + top_k: None, + ..request + }) +} + +pub fn map_openai_context_management_to_anthropic(context_management: &Value) -> Option { + match context_management { + Value::Object(edits) if edits.contains_key("edits") => Some(context_management.clone()), + Value::Array(entries) => { + let edits: Vec = entries + .iter() + .filter_map(Value::as_object) + .filter(|entry| entry.get("type").and_then(Value::as_str) == Some("compaction")) + .map(|entry| { + let trigger = entry.get("compact_threshold").and_then(Value::as_f64).map( + |threshold| json!({"type": "input_tokens", "value": threshold as i64}), + ); + let passthrough = entry + .iter() + .filter(|(key, _)| !matches!(key.as_str(), "type" | "compact_threshold")) + .map(|(key, value)| (key.clone(), value.clone())); + Value::Object( + [("type".to_string(), json!("compact_20260112"))] + .into_iter() + .chain(trigger.map(|trigger| ("trigger".to_string(), trigger))) + .chain(passthrough) + .collect::>(), + ) + }) + .collect(); + (!edits.is_empty()).then(|| json!({"edits": edits})) + } + _ => None, + } } pub fn non_empty(value: Option<&str>) -> Option<&str> { @@ -64,70 +239,619 @@ pub fn resolve_anthropic_api_base( api_base: Option<&str>, env_lookup: &dyn Fn(&str) -> Option, ) -> String { + let env = |name: &str| env_lookup(name).filter(|value| !value.trim().is_empty()); non_empty(api_base) .map(str::to_string) - .or_else(|| env_lookup(ANTHROPIC_API_BASE_ENV).filter(|value| !value.trim().is_empty())) + .or_else(|| env(ANTHROPIC_API_BASE_ENV)) + .or_else(|| env(ANTHROPIC_BASE_URL_ENV)) .unwrap_or_else(|| DEFAULT_ANTHROPIC_API_BASE.to_string()) } #[cfg(test)] mod tests { + use std::process::Command; + + use rstest::{fixture, rstest}; + use super::*; + use crate::anthropic::common_utils::{ENCRYPTED_REASONING_SIGNATURE_PREFIX, beta}; - #[test] - fn url_defaults_to_public_anthropic_endpoint() { + type Env = &'static [(&'static str, &'static str)]; + + const BOTH_BASE_ENVS: Env = &[ + (ANTHROPIC_API_BASE_ENV, "https://api-base.example.com"), + (ANTHROPIC_BASE_URL_ENV, "https://base-url.example.com"), + ]; + const API_KEY_ENV: Env = &[(ANTHROPIC_API_KEY_ENV, "sk-env")]; + const MISSING_API_KEY: &str = + "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY environment variable"; + const LOW_BUDGET_ENV: &str = "DEFAULT_REASONING_EFFORT_LOW_THINKING_BUDGET"; + const PROCESS_ENV_PROBE: &str = "LITELLM_MESSAGES_TRANSFORM_CONTEXT_PROBE"; + + fn merged(base: Value, fields: Value) -> Value { + Value::Object( + base.as_object() + .unwrap() + .clone() + .into_iter() + .chain(fields.as_object().unwrap().clone()) + .collect(), + ) + } + + fn body(fields: Value) -> Value { + merged( + json!({ + "model": "claude", + "max_tokens": 1024, + "messages": [{"role": "user", "content": "Hello"}] + }), + fields, + ) + } + + fn request(fields: Value) -> AnthropicMessagesRequest { + serde_json::from_value(body(fields)).unwrap() + } + + fn no_env(_: &str) -> Option { + None + } + + fn env(vars: Env) -> impl Fn(&str) -> Option { + move |name| { + vars.iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| value.to_string()) + } + } + + fn headers(pairs: &[(&str, &str)]) -> Headers { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect() + } + + fn transform( + fields: Value, + capabilities: AnthropicModelCapabilities, + drop_params: bool, + ) -> Result { + ANTHROPIC_MESSAGES_CONFIG + .transform_anthropic_messages_request( + request(fields), + &MessagesTransformContext::with_lookup(capabilities, drop_params, &no_env), + ) + .map(|transformed| serde_json::to_value(transformed).unwrap()) + } + + fn invalid(message: &str) -> Result { + Err(Error::InvalidRequest(message.to_string())) + } + + fn advisor_history() -> Value { + json!([ + {"role": "user", "content": "Build a worker pool."}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Let me consult the advisor."}, + {"type": "server_tool_use", "id": "srvtoolu_abc123", "name": "advisor", "input": {}}, + {"type": "advisor_tool_result", "tool_use_id": "srvtoolu_abc123", "content": {"type": "advisor_result", "text": "Use channels."}}, + {"type": "text", "text": "Here is the implementation."} + ]} + ]) + } + + #[fixture] + fn unmapped() -> AnthropicModelCapabilities { + AnthropicModelCapabilities::default() + } + + #[fixture] + fn sampling_removed() -> AnthropicModelCapabilities { + AnthropicModelCapabilities { + supports_sampling_params: false, + ..Default::default() + } + } + + #[fixture] + fn fast_mode() -> AnthropicModelCapabilities { + AnthropicModelCapabilities { + supports_speed: true, + ..Default::default() + } + } + + #[rstest] + #[case::alone(json!({"max_tokens": null}))] + #[case::ahead_of_the_param_gate(json!({"max_tokens": null, "speed": "fast"}))] + fn missing_max_tokens_is_rejected(#[case] fields: Value, unmapped: AnthropicModelCapabilities) { assert_eq!( - complete_anthropic_url(None, &|_| None), - "https://api.anthropic.com/v1/messages" + transform(fields, unmapped, false), + invalid("max_tokens is required for Anthropic /v1/messages API") + ); + } + + #[rstest] + #[case::sampling_params_on_a_sampling_model( + unmapped(), + false, + json!({"temperature": 0.3, "top_p": 0.9, "top_k": 40}) + )] + #[case::sampling_params_on_a_sampling_model_under_drop_params( + unmapped(), + true, + json!({"temperature": 0.3, "top_p": 0.9, "top_k": 40}) + )] + #[case::unit_temperature_on_a_sampling_removed_model( + sampling_removed(), + false, + json!({"temperature": 1.0}) + )] + #[case::unit_temperature_on_a_sampling_removed_model_under_drop_params( + sampling_removed(), + true, + json!({"temperature": 1.0}) + )] + #[case::speed_on_a_fast_mode_model(fast_mode(), false, json!({"speed": "fast"}))] + #[case::speed_on_a_fast_mode_model_under_drop_params(fast_mode(), true, json!({"speed": "fast"}))] + #[case::native_context_management_edits(unmapped(), false, json!({"context_management": {"edits": [{ + "type": "clear_tool_uses_20250919", + "trigger": {"type": "input_tokens", "value": 30000}, + "keep": {"type": "tool_uses", "value": 3}, + "clear_at_least": {"type": "input_tokens", "value": 5000}, + "exclude_tools": ["web_search"], + "clear_tool_inputs": false + }]}}))] + #[case::first_party_billing_header_system_block(unmapped(), false, json!({"system": [ + {"type": "text", "text": "x-anthropic-billing-header: cc_version=1"}, + {"type": "text", "text": "real system prompt"} + ]}))] + #[case::anthropic_signed_reasoning_history(unmapped(), false, json!({"messages": [ + {"role": "user", "content": "Solve it."}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": "EqQBCkYIAxgCIkA_anthropic_signed"}, + {"type": "redacted_thinking", "data": "EmwKAhgBEgy_anthropic_minted"}, + {"type": "text", "text": "The answer."} + ]} + ]}))] + #[case::advisor_history_alongside_the_advisor_tool(unmapped(), false, json!({ + "messages": advisor_history(), + "tools": [{"type": "advisor_20260301", "name": "advisor"}] + }))] + fn request_is_forwarded_unchanged( + #[case] capabilities: AnthropicModelCapabilities, + #[case] drop_params: bool, + #[case] fields: Value, + ) { + assert_eq!( + transform(fields.clone(), capabilities, drop_params), + Ok(body(fields)) + ); + } + + #[rstest] + #[case::temperature(sampling_removed(), json!({"temperature": 0.3}), json!({}))] + #[case::top_p(sampling_removed(), json!({"top_p": 0.9}), json!({}))] + #[case::top_k(sampling_removed(), json!({"top_k": 40}), json!({}))] + #[case::every_sampling_param_keeping_the_rest( + sampling_removed(), + json!({"temperature": 0.3, "top_p": 0.9, "top_k": 40, "stream": true}), + json!({"stream": true}) + )] + #[case::speed_on_a_sampling_model( + unmapped(), + json!({"speed": "fast", "temperature": 0.5}), + json!({"temperature": 0.5}) + )] + #[case::speed_on_a_sampling_removed_model( + sampling_removed(), + json!({"speed": "fast", "temperature": 1.0}), + json!({"temperature": 1.0}) + )] + fn removed_params_are_dropped_under_drop_params( + #[case] capabilities: AnthropicModelCapabilities, + #[case] fields: Value, + #[case] expected: Value, + ) { + assert_eq!(transform(fields, capabilities, true), Ok(body(expected))); + } + + #[rstest] + #[case::temperature( + sampling_removed(), + json!({"temperature": 0.3}), + "claude does not support temperature=0.3. Only temperature=1 is supported. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::temperature_just_below_one( + sampling_removed(), + json!({"temperature": 0.99}), + "claude does not support temperature=0.99. Only temperature=1 is supported. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::whole_number_temperature_keeps_its_decimal( + sampling_removed(), + json!({"temperature": 2.0}), + "claude does not support temperature=2.0. Only temperature=1 is supported. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::top_p( + sampling_removed(), + json!({"top_p": 0.9}), + "claude does not support top_p=0.9. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::top_k( + sampling_removed(), + json!({"top_k": 5}), + "claude does not support top_k=5. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::top_k_next_to_unit_temperature( + sampling_removed(), + json!({"temperature": 1.0, "top_k": 5}), + "claude does not support top_k=5. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::temperature_ahead_of_top_k( + sampling_removed(), + json!({"temperature": 0.5, "top_k": 5}), + "claude does not support temperature=0.5. Only temperature=1 is supported. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::top_p_ahead_of_top_k( + sampling_removed(), + json!({"top_p": 0.9, "top_k": 5}), + "claude does not support top_p=0.9. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::speed( + unmapped(), + json!({"speed": "fast"}), + "claude does not support speed='fast'. To drop unsupported params, set `litellm.drop_params = True`." + )] + #[case::speed_ahead_of_sampling_params( + sampling_removed(), + json!({"speed": "fast", "temperature": 0.5}), + "claude does not support speed='fast'. To drop unsupported params, set `litellm.drop_params = True`." + )] + fn removed_params_are_rejected_without_drop_params( + #[case] capabilities: AnthropicModelCapabilities, + #[case] fields: Value, + #[case] message: &str, + ) { + assert_eq!(transform(fields, capabilities, false), invalid(message)); + } + + #[rstest] + #[case::compaction_threshold( + json!([{"type": "compaction", "compact_threshold": 200000}]), + Some(json!({"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 200000}}]})) + )] + #[case::other_keys_pass_through( + json!([{"type": "compaction", "compact_threshold": 150000, "instructions": "Focus on preserving code snippets"}]), + Some(json!({"edits": [{ + "type": "compact_20260112", + "trigger": {"type": "input_tokens", "value": 150000}, + "instructions": "Focus on preserving code snippets" + }]})) + )] + #[case::float_threshold_is_truncated( + json!([{"type": "compaction", "compact_threshold": 150000.9}]), + Some(json!({"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]})) + )] + #[case::compaction_without_threshold( + json!([{"type": "compaction"}]), + Some(json!({"edits": [{"type": "compact_20260112"}]})) + )] + #[case::non_numeric_threshold_is_dropped( + json!([{"type": "compaction", "compact_threshold": "150000"}]), + Some(json!({"edits": [{"type": "compact_20260112"}]})) + )] + #[case::non_object_entries_are_skipped( + json!([42, "compaction", null, [], {"type": "compaction", "compact_threshold": 1000}]), + Some(json!({"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 1000}}]})) + )] + #[case::only_compaction_entries_are_mapped_in_order( + json!([ + {"type": "compaction", "compact_threshold": 1000}, + {"type": "other", "compact_threshold": 5}, + {"type": "compaction", "instructions": "second"} + ]), + Some(json!({"edits": [ + {"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 1000}}, + {"type": "compact_20260112", "instructions": "second"} + ]})) + )] + #[case::list_without_compaction(json!([{"type": "other"}]), None)] + #[case::empty_list(json!([]), None)] + #[case::anthropic_edits_pass_through( + json!({"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]}), + Some(json!({"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 150000}}]})) + )] + #[case::object_without_edits(json!({"type": "compaction"}), None)] + #[case::scalar(json!("compaction"), None)] + fn openai_context_management_maps_to_anthropic_edits( + #[case] context_management: Value, + #[case] expected: Option, + ) { + assert_eq!( + map_openai_context_management_to_anthropic(&context_management), + expected + ); + } + + #[rstest] + #[case::openai_list_is_mapped( + json!([{"type": "compaction", "compact_threshold": 200000}]), + json!({"edits": [{"type": "compact_20260112", "trigger": {"type": "input_tokens", "value": 200000}}]}) + )] + #[case::unmappable_list_is_kept(json!([{"type": "other"}]), json!([{"type": "other"}]))] + #[case::unmappable_object_is_kept(json!({"type": "other"}), json!({"type": "other"}))] + fn context_management_reaches_the_wire( + #[case] context_management: Value, + #[case] expected: Value, + unmapped: AnthropicModelCapabilities, + ) { + assert_eq!( + transform( + json!({"context_management": context_management}), + unmapped, + false + ), + Ok(body(json!({"context_management": expected}))) + ); + } + + #[rstest] + #[case::without_tools(json!({}))] + #[case::with_only_other_tools(json!({"tools": [{"name": "get_weather", "input_schema": {"type": "object"}}]}))] + fn advisor_history_is_stripped_without_the_advisor_tool( + #[case] tools: Value, + unmapped: AnthropicModelCapabilities, + ) { + let stripped = json!([ + {"role": "user", "content": "Build a worker pool."}, + {"role": "assistant", "content": [ + {"type": "text", "text": "Let me consult the advisor."}, + {"type": "text", "text": "Here is the implementation."} + ]} + ]); + assert_eq!( + transform( + merged(tools.clone(), json!({"messages": advisor_history()})), + unmapped, + false + ), + Ok(body(merged(tools, json!({"messages": stripped})))) + ); + } + + #[rstest] + fn bridge_minted_reasoning_is_stripped_from_the_wire(unmapped: AnthropicModelCapabilities) { + let messages = json!([ + {"role": "user", "content": "Solve it."}, + {"role": "assistant", "content": [ + {"type": "thinking", "thinking": "plan", "signature": format!("{ENCRYPTED_REASONING_SIGNATURE_PREFIX}gAAAA_1")}, + {"type": "redacted_thinking", "data": format!("{ENCRYPTED_REASONING_SIGNATURE_PREFIX}gAAAA_2")}, + {"type": "text", "text": "The answer."} + ]}, + {"role": "user", "content": "And the next one?"} + ]); + assert_eq!( + transform(json!({"messages": messages}), unmapped, false), + Ok(body(json!({"messages": [ + {"role": "user", "content": "Solve it."}, + {"role": "assistant", "content": [{"type": "text", "text": "The answer."}]}, + {"role": "user", "content": "And the next one?"} + ]}))) ); } #[test] - fn url_appends_messages_suffix_to_custom_base() { + fn thinking_is_translated_with_the_context_budgets() { + let context = MessagesTransformContext::with_lookup( + AnthropicModelCapabilities { + supports_reasoning: true, + ..Default::default() + }, + false, + &env(&[(LOW_BUDGET_ENV, "2000")]), + ); + let transformed = ANTHROPIC_MESSAGES_CONFIG + .transform_anthropic_messages_request( + request(json!({"max_tokens": 4096, "reasoning_effort": "low"})), + &context, + ) + .map(|transformed| serde_json::to_value(transformed).unwrap()); assert_eq!( - complete_anthropic_url(Some("https://proxy.internal"), &|_| None), - "https://proxy.internal/v1/messages" + transformed, + Ok(body(json!({ + "max_tokens": 4096, + "thinking": {"type": "enabled", "budget_tokens": 2000} + }))) ); } #[test] - fn url_leaves_complete_messages_endpoint_untouched() { + fn new_reads_thinking_budgets_from_the_process_environment() { + if std::env::var_os(PROCESS_ENV_PROBE).is_some() { + assert_eq!( + MessagesTransformContext::new(sampling_removed(), true), + MessagesTransformContext { + thinking: ThinkingContext { + capabilities: sampling_removed(), + budgets: ThinkingBudgets { + low: 2000, + ..ThinkingBudgets::default() + }, + }, + drop_params: true, + } + ); + return; + } + let (_, test_path) = concat!( + module_path!(), + "::new_reads_thinking_budgets_from_the_process_environment" + ) + .split_once("::") + .unwrap(); + let other_tiers = ["MINIMAL", "MEDIUM", "HIGH", "XHIGH", "MAX"] + .map(|tier| format!("DEFAULT_REASONING_EFFORT_{tier}_THINKING_BUDGET")); + let output = other_tiers + .iter() + .fold( + Command::new(std::env::current_exe().unwrap()), + |mut command, name| { + command.env_remove(name); + command + }, + ) + .args([test_path, "--exact"]) + .env(PROCESS_ENV_PROBE, "1") + .env(LOW_BUDGET_ENV, "2000") + .output() + .unwrap(); + let stdout = String::from_utf8_lossy(&output.stdout); + assert!( + output.status.success() && stdout.contains("1 passed"), + "{stdout}{}", + String::from_utf8_lossy(&output.stderr) + ); + } + + #[rstest] + #[case::public_endpoint_by_default(None, &[], "https://api.anthropic.com")] + #[case::explicit_api_base_beats_env( + Some("https://explicit.example.com"), + BOTH_BASE_ENVS, + "https://explicit.example.com" + )] + #[case::explicit_api_base_is_trimmed( + Some(" https://explicit.example.com "), + &[], + "https://explicit.example.com" + )] + #[case::blank_api_base_falls_back_to_env( + Some(" "), + BOTH_BASE_ENVS, + "https://api-base.example.com" + )] + #[case::api_base_env_beats_base_url_env(None, BOTH_BASE_ENVS, "https://api-base.example.com")] + #[case::base_url_env_without_api_base_env( + None, + &[(ANTHROPIC_BASE_URL_ENV, "https://base-url.example.com")], + "https://base-url.example.com" + )] + #[case::blank_api_base_env_falls_back_to_base_url_env( + None, + &[(ANTHROPIC_API_BASE_ENV, " \t "), (ANTHROPIC_BASE_URL_ENV, "https://base-url.example.com")], + "https://base-url.example.com" + )] + #[case::blank_envs_fall_back_to_public_endpoint( + None, + &[(ANTHROPIC_API_BASE_ENV, ""), (ANTHROPIC_BASE_URL_ENV, " ")], + "https://api.anthropic.com" + )] + fn api_base_resolution( + #[case] api_base: Option<&str>, + #[case] vars: Env, + #[case] expected: &str, + ) { + assert_eq!(resolve_anthropic_api_base(api_base, &env(vars)), expected); + } + + #[rstest] + #[case::public_endpoint(None, &[], "https://api.anthropic.com/v1/messages")] + #[case::base_url_env( + None, + &[(ANTHROPIC_BASE_URL_ENV, "https://custom.example.com")], + "https://custom.example.com/v1/messages" + )] + #[case::custom_base(Some("https://proxy.internal"), &[], "https://proxy.internal/v1/messages")] + #[case::trailing_slash(Some("https://proxy.internal/"), &[], "https://proxy.internal/v1/messages")] + #[case::complete_endpoint( + Some("https://proxy.internal/v1/messages"), + &[], + "https://proxy.internal/v1/messages" + )] + #[case::complete_endpoint_with_trailing_slash( + Some("https://proxy.internal/v1/messages/"), + &[], + "https://proxy.internal/v1/messages" + )] + fn complete_url_ends_in_the_messages_path( + #[case] api_base: Option<&str>, + #[case] vars: Env, + #[case] expected: &str, + ) { assert_eq!( - complete_anthropic_url(Some("https://proxy.internal/v1/messages"), &|_| None), - "https://proxy.internal/v1/messages" + ANTHROPIC_MESSAGES_CONFIG.get_complete_url(api_base, "claude", &env(vars)), + Ok(expected.to_string()) + ); + } + + #[rstest] + #[case::param_beats_env(Some("sk-param"), API_KEY_ENV, Ok("sk-param"))] + #[case::param_is_trimmed(Some(" sk-param "), &[], Ok("sk-param"))] + #[case::blank_param_falls_back_to_env(Some(" "), API_KEY_ENV, Ok("sk-env"))] + #[case::env_without_param(None, API_KEY_ENV, Ok("sk-env"))] + #[case::blank_env_is_missing(None, &[(ANTHROPIC_API_KEY_ENV, " ")], Err(MISSING_API_KEY))] + #[case::nothing_is_missing(None, &[], Err(MISSING_API_KEY))] + fn api_key_resolution( + #[case] api_key: Option<&str>, + #[case] vars: Env, + #[case] expected: Result<&str, &str>, + ) { + assert_eq!( + resolve_anthropic_api_key(api_key, &env(vars)).map_err(|error| error.to_string()), + expected.map(str::to_string).map_err(str::to_string) ); } #[test] - fn url_falls_back_to_env_base() { - let with_env = |key: &str| { - (key == ANTHROPIC_API_BASE_ENV).then(|| "https://env.anthropic".to_string()) - }; + fn config_reports_a_missing_key_as_an_auth_error() { assert_eq!( - complete_anthropic_url(Some(" "), &with_env), - "https://env.anthropic/v1/messages" + ANTHROPIC_MESSAGES_CONFIG.resolve_api_key(None, &no_env), + Err(Error::Auth(litellm_auth::Error::MissingApiKey { + provider: "Anthropic", + environment_variable: ANTHROPIC_API_KEY_ENV, + })) ); } #[test] - fn api_key_prefers_param_then_env_then_errors() { + fn config_authenticates_with_the_anthropic_auth_token() { assert_eq!( - resolve_anthropic_api_key(Some("sk-param"), &|_| None).unwrap(), - "sk-param" + ANTHROPIC_MESSAGES_CONFIG.authenticate( + vec![], + None, + &env(&[("ANTHROPIC_AUTH_TOKEN", "auth-token")]) + ), + Ok(headers(&[("authorization", "Bearer auth-token")])) ); - let with_env = |key: &str| (key == ANTHROPIC_API_KEY_ENV).then(|| "sk-env".to_string()); + } + + #[test] + fn config_requests_the_betas_the_request_features_need() { assert_eq!( - resolve_anthropic_api_key(Some(" "), &with_env).unwrap(), - "sk-env" - ); - assert_eq!( - resolve_anthropic_api_key(None, &|_| None) - .expect_err("missing key") - .to_string(), - "Missing Anthropic API Key - Set `api_key` or the ANTHROPIC_API_KEY environment variable" + ANTHROPIC_MESSAGES_CONFIG.request_headers( + headers(&[("x-api-key", "sk")]), + &request(json!({"speed": "fast"})) + ), + headers(&[ + ("x-api-key", "sk"), + ("anthropic-beta", beta::FAST_MODE_2026_02_01) + ]) ); } + #[rstest] + #[case::absent(None, None)] + #[case::blank(Some(" \t "), None)] + #[case::padded(Some(" value "), Some("value"))] + fn non_empty_trims_and_drops_blank_values( + #[case] value: Option<&str>, + #[case] expected: Option<&str>, + ) { + assert_eq!(non_empty(value), expected); + } + #[test] fn auth_strategy_and_default_headers_match_anthropic() { assert_eq!( @@ -142,4 +866,26 @@ mod tests { ] ); } + + #[test] + fn secret_names_cover_every_credential_and_base_lookup() { + let requested = std::cell::RefCell::new(Vec::::new()); + let record = |name: &str| -> Option { + requested.borrow_mut().push(name.to_string()); + None + }; + let _ = ANTHROPIC_MESSAGES_CONFIG.authenticate(Vec::new(), None, &record); + let _ = ANTHROPIC_MESSAGES_CONFIG.get_complete_url(None, "claude", &record); + let requested = requested.into_inner(); + assert!(!requested.is_empty()); + let undeclared: Vec<&String> = requested + .iter() + .filter(|name| { + !ANTHROPIC_MESSAGES_CONFIG + .secret_names() + .contains(&name.as_str()) + }) + .collect(); + assert_eq!(undeclared, Vec::<&String>::new()); + } } diff --git a/litellm-rust/crates/llms/src/anthropic/mod.rs b/litellm-rust/crates/llms/src/anthropic/mod.rs index d181ceaca3c..755bc7d1907 100644 --- a/litellm-rust/crates/llms/src/anthropic/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/mod.rs @@ -1,5 +1,6 @@ pub mod batches; pub mod chat; +pub mod common_utils; pub mod count_tokens; pub mod experimental_pass_through; diff --git a/litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs b/litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs index 99f55f18afc..c409f7f687e 100644 --- a/litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs +++ b/litellm-rust/crates/llms/src/azure_ai/anthropic/messages_transformation.rs @@ -4,14 +4,15 @@ use litellm_types::llms::anthropic_messages::{ }, anthropic_response::AnthropicMessagesResponse, }; -use serde_json::{Map, Value}; use crate::{ anthropic::experimental_pass_through::messages::transformation::{ ANTHROPIC_MESSAGES_CONFIG, AnthropicMessagesConfig, non_empty, }, base_llm::{ - anthropic_messages::transformation::{BaseAnthropicMessagesConfig, MessagesAuthStrategy}, + anthropic_messages::transformation::{ + BaseAnthropicMessagesConfig, Headers, MessagesAuthStrategy, MessagesTransformContext, + }, chat::transformation::Error, }, }; @@ -21,7 +22,6 @@ const AZURE_API_BASE_ENV: &str = "AZURE_API_BASE"; const ANTHROPIC_PATH_SEGMENT: &str = "/anthropic"; const MESSAGES_PATH_SUFFIX: &str = "/v1/messages"; const SYSTEM_ROLE: &str = "system"; -const TEXT_BLOCK_TYPE: &str = "text"; pub struct AzureAnthropicMessagesConfig { anthropic: AnthropicMessagesConfig, @@ -45,6 +45,7 @@ impl BaseAnthropicMessagesConfig for AzureAnthropicMessagesConfig { fn transform_anthropic_messages_request( &self, request: AnthropicMessagesRequest, + context: &MessagesTransformContext, ) -> Result { let mut request = fold_system_role_messages(request); if let Some(system) = request.system.as_mut() { @@ -54,7 +55,8 @@ impl BaseAnthropicMessagesConfig for AzureAnthropicMessagesConfig { .messages .iter_mut() .for_each(strip_scope_from_message); - self.anthropic.transform_anthropic_messages_request(request) + self.anthropic + .transform_anthropic_messages_request(request, context) } fn transform_anthropic_messages_response( @@ -74,6 +76,10 @@ impl BaseAnthropicMessagesConfig for AzureAnthropicMessagesConfig { resolve_azure_api_key(api_key, env_lookup) } + fn secret_names(&self) -> &'static [&'static str] { + &[AZURE_API_KEY_ENV, AZURE_API_BASE_ENV] + } + fn auth_strategy(&self) -> MessagesAuthStrategy { self.anthropic.auth_strategy() } @@ -85,6 +91,10 @@ impl BaseAnthropicMessagesConfig for AzureAnthropicMessagesConfig { fn default_headers(&self) -> &'static [(&'static str, &'static str)] { self.anthropic.default_headers() } + + fn request_headers(&self, headers: Headers, request: &AnthropicMessagesRequest) -> Headers { + self.anthropic.request_headers(headers, request) + } } pub fn resolve_azure_api_key( @@ -143,17 +153,7 @@ fn strip_scope_from_message(message: &mut AnthropicMessage) { } fn text_content_block(text: String) -> ContentBlock { - let extra = Map::from_iter([ - ( - "type".to_string(), - Value::String(TEXT_BLOCK_TYPE.to_string()), - ), - ("text".to_string(), Value::String(text)), - ]); - ContentBlock { - cache_control: None, - extra, - } + ContentBlock::text(text) } fn content_into_blocks(content: MessageContent) -> Vec { @@ -202,6 +202,7 @@ mod tests { use serde_json::json; use super::*; + use crate::anthropic::common_utils::AnthropicModelCapabilities; fn request_from(value: serde_json::Value) -> AnthropicMessagesRequest { serde_json::from_value(value).expect("valid request") @@ -346,7 +347,7 @@ mod tests { let transformed = to_value( AZURE_ANTHROPIC_MESSAGES_CONFIG - .transform_anthropic_messages_request(request) + .transform_anthropic_messages_request(request, &MessagesTransformContext::default()) .expect("request transforms"), ); @@ -373,10 +374,13 @@ mod tests { "messages": [{"role": "user", "content": "hi"}] })); let once = AZURE_ANTHROPIC_MESSAGES_CONFIG - .transform_anthropic_messages_request(request) + .transform_anthropic_messages_request(request, &MessagesTransformContext::default()) .expect("request transforms"); let twice = AZURE_ANTHROPIC_MESSAGES_CONFIG - .transform_anthropic_messages_request(once.clone()) + .transform_anthropic_messages_request( + once.clone(), + &MessagesTransformContext::default(), + ) .expect("request transforms"); assert_eq!(once, twice); assert_eq!(to_value(once)["system"], json!("plain string system")); @@ -408,9 +412,21 @@ mod tests { "inference_geo": "us", "litellm_metadata": {"trace": "abc"} }); + let context = MessagesTransformContext::with_lookup( + AnthropicModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_legacy_thinking: true, + supports_output_config: true, + supports_speed: true, + ..Default::default() + }, + false, + &|_: &str| None, + ); let transformed = to_value( AZURE_ANTHROPIC_MESSAGES_CONFIG - .transform_anthropic_messages_request(request_from(body.clone())) + .transform_anthropic_messages_request(request_from(body.clone()), &context) .expect("request transforms"), ); assert_eq!(transformed, body); @@ -430,7 +446,7 @@ mod tests { let transformed = to_value( AZURE_ANTHROPIC_MESSAGES_CONFIG - .transform_anthropic_messages_request(request) + .transform_anthropic_messages_request(request, &MessagesTransformContext::default()) .expect("request transforms"), ); @@ -460,7 +476,7 @@ mod tests { let transformed = to_value( AZURE_ANTHROPIC_MESSAGES_CONFIG - .transform_anthropic_messages_request(request) + .transform_anthropic_messages_request(request, &MessagesTransformContext::default()) .expect("request transforms"), ); @@ -485,9 +501,21 @@ mod tests { {"role": "assistant", "content": "hello"} ] }); + let context = MessagesTransformContext::with_lookup( + AnthropicModelCapabilities { + supports_reasoning: true, + supports_adaptive_thinking: true, + supports_legacy_thinking: true, + supports_output_config: true, + supports_speed: true, + ..Default::default() + }, + false, + &|_: &str| None, + ); let transformed = to_value( AZURE_ANTHROPIC_MESSAGES_CONFIG - .transform_anthropic_messages_request(request_from(body.clone())) + .transform_anthropic_messages_request(request_from(body.clone()), &context) .expect("request transforms"), ); assert_eq!(transformed, body); @@ -500,6 +528,57 @@ mod tests { assert!(err.is_data()); } + #[rstest::rstest] + #[case::compact_context_management_edit( + json!({"context_management": {"edits": [{"type": "compact_20260112"}]}}), + &[], + &[("x-api-key", "k"), ("anthropic-beta", "compact-2026-01-12")] + )] + #[case::forwarded_beta_merged_with_structured_output( + json!({"output_config": {"format": {"type": "json_schema"}}}), + &[("anthropic-beta", "web-search-2025-03-05")], + &[("x-api-key", "k"), ("anthropic-beta", "structured-outputs-2025-11-13,web-search-2025-03-05")] + )] + #[case::no_feature_needs_a_beta(json!({}), &[], &[("x-api-key", "k")])] + fn request_headers_carry_the_anthropic_feature_betas( + #[case] fields: serde_json::Value, + #[case] forwarded: &[(&str, &str)], + #[case] expected: &[(&str, &str)], + ) { + let pairs = |pairs: &[(&str, &str)]| -> Vec<(String, String)> { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect() + }; + let serde_json::Value::Object(fields) = fields else { + panic!("case fields are an object") + }; + let request = request_from(serde_json::Value::Object( + [ + ("model".to_string(), json!("claude-sonnet")), + ("max_tokens".to_string(), json!(16)), + ( + "messages".to_string(), + json!([{"role": "user", "content": "hi"}]), + ), + ] + .into_iter() + .chain(fields) + .collect(), + )); + assert_eq!( + AZURE_ANTHROPIC_MESSAGES_CONFIG.request_headers( + pairs(&[("x-api-key", "k")]) + .into_iter() + .chain(pairs(forwarded)) + .collect(), + &request + ), + pairs(expected) + ); + } + #[test] fn transform_response_passes_through() { let response: AnthropicMessagesResponse = serde_json::from_value(json!({ @@ -521,4 +600,26 @@ mod tests { assert_eq!(value["stop_sequence"], json!(null)); assert_eq!(value["content"][0]["text"], json!("hello")); } + + #[test] + fn secret_names_cover_every_credential_and_base_lookup() { + let requested = std::cell::RefCell::new(Vec::::new()); + let record = |name: &str| -> Option { + requested.borrow_mut().push(name.to_string()); + None + }; + let _ = AZURE_ANTHROPIC_MESSAGES_CONFIG.authenticate(Vec::new(), None, &record); + let _ = AZURE_ANTHROPIC_MESSAGES_CONFIG.get_complete_url(None, "claude", &record); + let requested = requested.into_inner(); + assert!(!requested.is_empty()); + let undeclared: Vec<&String> = requested + .iter() + .filter(|name| { + !AZURE_ANTHROPIC_MESSAGES_CONFIG + .secret_names() + .contains(&name.as_str()) + }) + .collect(); + assert_eq!(undeclared, Vec::<&String>::new()); + } } diff --git a/litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs index 5b4afb601d2..8db14687214 100644 --- a/litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/anthropic_messages/transformation.rs @@ -1,8 +1,14 @@ +use litellm_http::request::{has_bearer_auth, has_header}; use litellm_types::llms::anthropic_messages::{ anthropic_request::AnthropicMessagesRequest, anthropic_response::AnthropicMessagesResponse, }; -use crate::base_llm::chat::transformation::Error; +use crate::{ + anthropic::experimental_pass_through::messages::thinking::ThinkingContext, + base_llm::chat::transformation::Error, +}; + +pub type Headers = Vec<(String, String)>; #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub enum MessagesAuthStrategy { @@ -19,6 +25,12 @@ impl MessagesAuthStrategy { } } +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct MessagesTransformContext { + pub thinking: ThinkingContext, + pub drop_params: bool, +} + pub trait BaseAnthropicMessagesConfig: Sync { fn get_complete_url( &self, @@ -30,6 +42,7 @@ pub trait BaseAnthropicMessagesConfig: Sync { fn transform_anthropic_messages_request( &self, request: AnthropicMessagesRequest, + _context: &MessagesTransformContext, ) -> Result { Ok(request) } @@ -48,6 +61,8 @@ pub trait BaseAnthropicMessagesConfig: Sync { env_lookup: &dyn Fn(&str) -> Option, ) -> Result; + fn secret_names(&self) -> &'static [&'static str]; + fn auth_strategy(&self) -> MessagesAuthStrategy { MessagesAuthStrategy::Header("x-api-key") } @@ -56,10 +71,225 @@ pub trait BaseAnthropicMessagesConfig: Sync { false } + fn authenticate( + &self, + headers: Headers, + api_key: Option<&str>, + env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let strategy = self.auth_strategy(); + if has_header(&headers, strategy.header_name()) + || (self.accepts_bearer_auth() && has_bearer_auth(&headers)) + { + return Ok(headers); + } + let api_key = self.resolve_api_key(api_key, env_lookup)?; + let auth_header = match strategy { + MessagesAuthStrategy::Bearer => { + ("authorization".to_string(), format!("Bearer {api_key}")) + } + MessagesAuthStrategy::Header(name) => (name.to_string(), api_key), + }; + Ok(headers.into_iter().chain([auth_header]).collect()) + } + fn default_headers(&self) -> &'static [(&'static str, &'static str)] { &[ ("anthropic-version", "2023-06-01"), ("content-type", "application/json"), ] } + + fn request_headers(&self, headers: Headers, _request: &AnthropicMessagesRequest) -> Headers { + headers + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + const X_API_KEY: MessagesAuthStrategy = MessagesAuthStrategy::Header("x-api-key"); + + struct StubConfig { + strategy: MessagesAuthStrategy, + accepts_bearer: bool, + } + + impl BaseAnthropicMessagesConfig for StubConfig { + fn secret_names(&self) -> &'static [&'static str] { + &[] + } + + fn get_complete_url( + &self, + _api_base: Option<&str>, + _model: &str, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + Ok(String::new()) + } + + fn resolve_api_key( + &self, + api_key: Option<&str>, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + api_key + .map(str::to_string) + .ok_or(Error::MissingField("api_key")) + } + + fn auth_strategy(&self) -> MessagesAuthStrategy { + self.strategy + } + + fn accepts_bearer_auth(&self) -> bool { + self.accepts_bearer + } + } + + struct DefaultsConfig; + + impl BaseAnthropicMessagesConfig for DefaultsConfig { + fn secret_names(&self) -> &'static [&'static str] { + &[] + } + + fn get_complete_url( + &self, + _api_base: Option<&str>, + _model: &str, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + Ok(String::new()) + } + + fn resolve_api_key( + &self, + api_key: Option<&str>, + _env_lookup: &dyn Fn(&str) -> Option, + ) -> Result { + api_key + .map(str::to_string) + .ok_or(Error::MissingField("api_key")) + } + } + + #[test] + fn default_config_adds_its_key_next_to_a_forwarded_bearer() { + assert_eq!( + DefaultsConfig.authenticate( + headers(&[("authorization", "Bearer forwarded")]), + Some("sk"), + &|_| None + ), + Ok(headers(&[ + ("authorization", "Bearer forwarded"), + ("x-api-key", "sk") + ])) + ); + } + + #[test] + fn default_request_headers_are_the_given_headers() { + let request: AnthropicMessagesRequest = serde_json::from_value(serde_json::json!({ + "model": "claude", + "max_tokens": 16, + "speed": "fast", + "messages": [{"role": "user", "content": "hi"}] + })) + .unwrap(); + assert_eq!( + DefaultsConfig.request_headers(headers(&[("x-api-key", "sk")]), &request), + headers(&[("x-api-key", "sk")]) + ); + } + + fn headers(pairs: &[(&str, &str)]) -> Headers { + pairs + .iter() + .map(|(name, value)| (name.to_string(), value.to_string())) + .collect() + } + + #[rstest] + #[case::own_header_is_kept( + X_API_KEY, + false, + headers(&[("x-api-key", "forwarded")]), + None, + Ok(headers(&[("x-api-key", "forwarded")])) + )] + #[case::own_header_in_any_casing_is_kept( + X_API_KEY, + false, + headers(&[("X-Api-Key", "forwarded")]), + None, + Ok(headers(&[("X-Api-Key", "forwarded")])) + )] + #[case::accepted_bearer_is_kept( + X_API_KEY, + true, + headers(&[("authorization", "Bearer forwarded")]), + None, + Ok(headers(&[("authorization", "Bearer forwarded")])) + )] + #[case::bearer_the_provider_does_not_accept_gets_the_key_too( + X_API_KEY, + false, + headers(&[("authorization", "Bearer forwarded")]), + Some("sk"), + Ok(headers(&[("authorization", "Bearer forwarded"), ("x-api-key", "sk")])) + )] + #[case::blank_bearer_gets_the_key( + X_API_KEY, + true, + headers(&[("authorization", "Bearer ")]), + Some("sk"), + Ok(headers(&[("authorization", "Bearer "), ("x-api-key", "sk")])) + )] + #[case::key_goes_in_the_provider_header( + X_API_KEY, + false, + headers(&[("content-type", "application/json")]), + Some("sk"), + Ok(headers(&[("content-type", "application/json"), ("x-api-key", "sk")])) + )] + #[case::key_goes_in_a_bearer( + MessagesAuthStrategy::Bearer, + false, + headers(&[]), + Some("sk"), + Ok(headers(&[("authorization", "Bearer sk")])) + )] + #[case::bearer_strategy_keeps_a_forwarded_authorization( + MessagesAuthStrategy::Bearer, + false, + headers(&[("authorization", "Bearer forwarded")]), + None, + Ok(headers(&[("authorization", "Bearer forwarded")])) + )] + #[case::missing_key_is_an_error( + X_API_KEY, + false, + headers(&[]), + None, + Err(Error::MissingField("api_key")) + )] + fn default_authenticate_applies_the_key_unless_a_credential_is_forwarded( + #[case] strategy: MessagesAuthStrategy, + #[case] accepts_bearer: bool, + #[case] forwarded: Headers, + #[case] api_key: Option<&str>, + #[case] expected: Result, + ) { + let config = StubConfig { + strategy, + accepts_bearer, + }; + assert_eq!(config.authenticate(forwarded, api_key, &|_| None), expected); + } } diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs index 1a9b170f661..9d97094aeda 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/host.rs @@ -2,9 +2,11 @@ use bytes::Bytes; use litellm_core::messages::{ Error, route::{Messages, MessagesCall, MessagesOp, MessagesOpResult, MessagesOutput}, + types::MessagesShaping, }; use litellm_host_python::{InvokeError, RouteHost, from_py, lookup, to_py}; use litellm_http::transport::Error as TransportError; +use litellm_types::utils::ProviderSpecificHeaders; use pyo3::{ exceptions::{PyException, PyValueError}, gc::{PyTraverseError, PyVisit}, @@ -18,9 +20,10 @@ use crate::{ marshal::{optional_timeout, python_timeout_seconds}, }; -/// The Anthropic Messages body fields a caller may pass besides `model` and `messages`, -/// as `AnthropicMessagesRequestOptionalParams` declares them. -const BODY_FIELDS: [&str; 20] = [ +const ROUTE_HOST_MODULE: &str = "litellm.rust_bridge.messages.route_host"; +const REQUEST_ERROR_MARKER: &str = "messages_request_error"; + +const BODY_FIELDS: [&str; 22] = [ "max_tokens", "metadata", "stop_sequences", @@ -35,14 +38,46 @@ const BODY_FIELDS: [&str; 20] = [ "top_p", "mcp_servers", "context_management", + "compaction", "container", "output_format", "speed", "output_config", "cache_control", "reasoning_effort", + "safeguards", ]; +fn merge_headers( + forwarded: Option>, + extra_headers: Option>, +) -> Option> { + let merged: Map = forwarded + .into_iter() + .flatten() + .chain(extra_headers.into_iter().flatten()) + .collect(); + (!merged.is_empty()).then_some(merged) +} + +fn native_error(py: Python<'_>, error: Error) -> PyResult { + match error { + Error::Transport(TransportError::Http { status, body }) => { + let error = RustUpstreamError::new_err((status, body)); + error + .value(py) + .setattr("headers", Vec::<(String, String)>::new())?; + Ok(error) + } + Error::InvalidRequest(message) => { + let error = PyValueError::new_err(message); + error.value(py).setattr(REQUEST_ERROR_MARKER, true)?; + Ok(error) + } + other => Ok(messages_error_to_pyerr(other)), + } +} + /// The Python side of the Messages route: projects the prepared arguments and builds the /// public response, chunks and exceptions. pub(super) struct MessagesRouteHost { @@ -84,19 +119,65 @@ impl MessagesRouteHost { .map(|value| python_timeout_seconds(py, value.unbind())) .transpose()? .flatten(); + let custom_llm_provider = string("custom_llm_provider")?; + let shaping = self.shaping(py, &model, custom_llm_provider.as_deref(), arguments)?; Ok(MessagesCall { model, body, api_key: string("api_key")?, api_base: string("api_base")?, - custom_llm_provider: string("custom_llm_provider")?, - extra_headers: argument("extra_headers")? - .map(|value| from_py(&value)) - .transpose()?, + extra_headers: self.merged_headers(py, arguments)?, + provider_specific_header: self.provider_specific_header(py, arguments)?, + custom_llm_provider, timeout: optional_timeout(timeout), + shaping, }) } + fn merged_headers( + &self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + ) -> PyResult>> { + let request = self.request.bind(py); + let mapping = |name: &str| -> PyResult>> { + lookup(arguments, request, name)? + .filter(|value| !value.is_none()) + .map(|value| from_py(&value)) + .transpose() + }; + Ok(merge_headers( + mapping("headers")?, + mapping("extra_headers")?, + )) + } + + fn provider_specific_header( + &self, + py: Python<'_>, + arguments: &Bound<'_, PyDict>, + ) -> PyResult> { + lookup(arguments, self.request.bind(py), "provider_specific_header")? + .filter(|value| !value.is_none()) + .map(|value| from_py(&value)) + .transpose() + } + + fn shaping( + &self, + py: Python<'_>, + model: &str, + custom_llm_provider: Option<&str>, + arguments: &Bound<'_, PyDict>, + ) -> PyResult { + let projected = py.import(ROUTE_HOST_MODULE)?.getattr("shaping")?.call1(( + model, + custom_llm_provider, + arguments, + ))?; + from_py(&projected) + } + fn provider(&self, py: Python<'_>) -> String { self.request .bind(py) @@ -112,7 +193,7 @@ impl MessagesRouteHost { return error; } let mapped = py - .import("litellm.rust_bridge.messages.route_host") + .import(ROUTE_HOST_MODULE) .and_then(|module| module.getattr("map_failure")) .and_then(|map| map.call1((error.value(py), self.request.bind(py), self.provider(py)))) .and_then(|mapped| { @@ -148,7 +229,7 @@ impl RouteHost for MessagesRouteHost { fn complete(&mut self, py: Python<'_>, response: MessagesOutput) -> PyResult> { match response { MessagesOutput::Message(message) => py - .import("litellm.rust_bridge.messages.route_host")? + .import(ROUTE_HOST_MODULE)? .getattr("response")? .call1((to_py(py, message.as_ref())?,)) .map(Bound::unbind), @@ -161,17 +242,12 @@ impl RouteHost for MessagesRouteHost { } fn classify(&self, py: Python<'_>, error: Error) -> PyResult { - let native = match error { - Error::Transport(TransportError::Http { status, body }) => { - let error = RustUpstreamError::new_err((status, body)); - error - .value(py) - .setattr("headers", Vec::<(String, String)>::new())?; - error - } - other => messages_error_to_pyerr(other), - }; - Ok(self.map_failure(py, native)) + if let Error::Secret(source) = &error + && let Some(original) = crate::secrets::python_error(py, source.source_error()) + { + return Ok(original); + } + Ok(self.map_failure(py, native_error(py, error)?)) } fn host_error(error: &PyErr) -> Error { @@ -184,3 +260,62 @@ impl RouteHost for MessagesRouteHost { visit.call(&self.request) } } + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn map(value: Value) -> Map { + serde_json::from_value(value).unwrap() + } + + #[rstest] + #[case::extra_over_forwarded( + Some(json!({"X-Priority": "forwarded", "X-Forwarded-Only": "keep"})), + Some(json!({"X-Priority": "extra", "X-Extra-Only": "also-keep"})), + Some(json!({"X-Priority": "extra", "X-Forwarded-Only": "keep", "X-Extra-Only": "also-keep"})), + )] + #[case::only_forwarded(Some(json!({"X-Forwarded": "yes"})), None, Some(json!({"X-Forwarded": "yes"})))] + #[case::only_extra_headers( + None, + Some(json!({"X-Custom-Header": "from-kwargs", "X-Auth-Token": "token123"})), + Some(json!({"X-Custom-Header": "from-kwargs", "X-Auth-Token": "token123"})), + )] + #[case::nothing(None, Some(json!({})), None)] + fn headers_merge_forwarded_then_extra( + #[case] forwarded: Option, + #[case] extra_headers: Option, + #[case] expected: Option, + ) { + assert_eq!( + merge_headers(forwarded.map(map), extra_headers.map(map)), + expected.map(map) + ); + } + + #[rstest] + #[case::rejected_request(Error::InvalidRequest("does not support top_k=5".into()), true)] + #[case::unresolvable_provider(Error::InvalidProvider("openai".into()), false)] + #[case::upstream_failure( + Error::Transport(TransportError::Http { status: 400, body: "bad".into() }), + false, + )] + fn only_request_rejections_carry_the_request_error_marker( + #[case] error: Error, + #[case] marked: bool, + ) { + Python::initialize(); + Python::attach(|py| { + let native = native_error(py, error).unwrap(); + let marker = native + .value(py) + .getattr_opt(REQUEST_ERROR_MARKER) + .unwrap() + .map(|value| value.extract::().unwrap()); + assert_eq!(marker.unwrap_or(false), marked); + }); + } +} diff --git a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs index 804d883e9ae..fd474e6b2d4 100644 --- a/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/messages/mod.rs @@ -39,11 +39,12 @@ fn run_messages( "the Rust Messages route does not serve this provider", )); } + let secrets = crate::secrets::source(py)?; run_legacy_call( py, SURFACE, PublicCall::capture(&request, &args, &kwargs)?, - crate::logger::LoggedMachine::new(messages_machine()), + crate::logger::LoggedMachine::new(messages_machine(secrets)), MessagesRouteHost::new(request.unbind()), asynchronous, ) diff --git a/litellm-rust/crates/types/Cargo.toml b/litellm-rust/crates/types/Cargo.toml index 6a2efa90ab4..0a0927386f0 100644 --- a/litellm-rust/crates/types/Cargo.toml +++ b/litellm-rust/crates/types/Cargo.toml @@ -8,3 +8,6 @@ repository.workspace = true [dependencies] serde.workspace = true serde_json.workspace = true + +[dev-dependencies] +rstest.workspace = true diff --git a/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs index 50eedf7ba09..2f7a75ba517 100644 --- a/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs +++ b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_request.rs @@ -17,12 +17,48 @@ pub enum MessageContent { #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct ContentBlock { + #[serde(rename = "type", default, skip_serializing_if = "Option::is_none")] + pub block_type: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub text: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signature: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub data: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub name: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_use_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option, #[serde(skip_serializing_if = "Option::is_none")] pub cache_control: Option, #[serde(flatten)] pub extra: Map, } +impl ContentBlock { + pub fn text(text: impl Into) -> Self { + Self { + block_type: Some("text".to_string()), + text: Some(text.into()), + ..Self::default() + } + } + + pub fn is_type(&self, block_type: &str) -> bool { + self.block_type.as_deref() == Some(block_type) + } +} + #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] pub struct CacheControl { #[serde(rename = "type", skip_serializing_if = "Option::is_none")] @@ -85,6 +121,126 @@ pub struct AnthropicMessagesRequest { pub speed: Option, #[serde(skip_serializing_if = "Option::is_none")] pub inference_geo: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_effort: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub compaction: Option, #[serde(flatten)] pub extra: Map, } + +impl AnthropicMessage { + pub fn blocks(&self) -> &[ContentBlock] { + match &self.content { + MessageContent::Blocks(blocks) => blocks, + MessageContent::Text(_) => &[], + } + } + + pub fn with_blocks(self, blocks: Vec) -> Self { + Self { + content: MessageContent::Blocks(blocks), + ..self + } + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn round_trip(value: &Value) -> Value { + let parsed: T = serde_json::from_value(value.clone()).unwrap(); + serde_json::to_value(parsed).unwrap() + } + + #[rstest] + #[case::text(json!({"type": "text", "text": "hi"}))] + #[case::text_with_citations_and_cache_control(json!({ + "type": "text", + "text": "hi", + "citations": [{"type": "char_location", "cited_text": "x"}], + "cache_control": {"type": "ephemeral", "ttl": "1h", "scope": "global", "future": 1} + }))] + #[case::image(json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "AA=="}}))] + #[case::thinking(json!({"type": "thinking", "thinking": "hmm", "signature": "sig"}))] + #[case::redacted_thinking(json!({"type": "redacted_thinking", "data": "opaque"}))] + #[case::tool_use(json!({"type": "tool_use", "id": "toolu_1", "name": "f", "input": {"q": [1, null]}}))] + #[case::tool_result_with_text(json!({"type": "tool_result", "tool_use_id": "toolu_1", "content": "ok", "is_error": false}))] + #[case::tool_result_with_blocks(json!({"type": "tool_result", "tool_use_id": "toolu_1", "content": [{"type": "text", "text": "ok"}]}))] + #[case::web_search_result_with_nulls(json!({ + "type": "web_search_tool_result", + "tool_use_id": "srvtoolu_1", + "content": [{"type": "web_search_result", "url": "u", "page_age": null, "encrypted_content": ""}] + }))] + #[case::provider_specific_fields(json!({"type": "tool_use", "id": "t", "name": "f", "input": {}, "provider_specific_fields": {"x": 1}}))] + #[case::untyped(json!({"unknown": {"nested": true}}))] + fn content_block_round_trips_unchanged(#[case] block: Value) { + assert_eq!(round_trip::(&block), block); + } + + #[test] + fn text_constructor_serializes_as_a_text_block() { + assert_eq!( + serde_json::to_value(ContentBlock::text("hello")).unwrap(), + json!({"type": "text", "text": "hello"}) + ); + } + + #[rstest] + #[case::same_type(json!({"type": "tool_use"}), "tool_use", true)] + #[case::other_type(json!({"type": "tool_result"}), "tool_use", false)] + #[case::prefix_of_type(json!({"type": "tool_use"}), "tool", false)] + #[case::no_type(json!({"text": "x"}), "text", false)] + fn is_type_matches_the_exact_block_type( + #[case] block: Value, + #[case] block_type: &str, + #[case] expected: bool, + ) { + let block: ContentBlock = serde_json::from_value(block).unwrap(); + assert_eq!(block.is_type(block_type), expected); + } + + #[rstest] + #[case::string_content(json!({"role": "user", "content": "hi"}), vec![])] + #[case::block_content( + json!({"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}), + vec![ContentBlock::text("a"), ContentBlock::text("b")], + )] + fn message_blocks_list_only_block_content( + #[case] message: Value, + #[case] expected: Vec, + ) { + let message: AnthropicMessage = serde_json::from_value(message).unwrap(); + assert_eq!(message.blocks(), expected.as_slice()); + } + + #[rstest] + #[case::replaces_string_content(json!({"role": "assistant", "content": "old", "name": "kept"}))] + #[case::replaces_block_content(json!({"role": "assistant", "content": [{"type": "text", "text": "old"}], "name": "kept"}))] + fn with_blocks_replaces_content_and_keeps_the_rest(#[case] message: Value) { + let message: AnthropicMessage = serde_json::from_value(message).unwrap(); + assert_eq!( + serde_json::to_value(message.with_blocks(vec![ContentBlock::text("new")])).unwrap(), + json!({"role": "assistant", "content": [{"type": "text", "text": "new"}], "name": "kept"}) + ); + } + + #[rstest] + #[case::minimal(json!({"model": "m", "messages": [{"role": "user", "content": "hi"}]}))] + #[case::reasoning_effort_compaction_and_unknown_fields(json!({ + "model": "m", + "messages": [{"role": "user", "content": [{"type": "text", "text": "hi"}]}], + "max_tokens": 8, + "reasoning_effort": "high", + "compaction": {"type": "auto"}, + "safeguards": [{"type": "dangerous_tool_use", "classifier_context": {"v": 1}}], + "metadata": {"user_id": "u"} + }))] + fn request_round_trips_unchanged(#[case] request: Value) { + assert_eq!(round_trip::(&request), request); + } +} diff --git a/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs index 0c3876aac59..0a2653f352f 100644 --- a/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs +++ b/litellm-rust/crates/types/src/llms/anthropic_messages/anthropic_response.rs @@ -9,8 +9,6 @@ pub struct AnthropicMessagesResponse { pub role: String, pub model: String, pub content: Vec, - // Anthropic always includes stop_reason / stop_sequence, null until the turn - // ends; serialize them even when None so callers see the same shape as Python. pub stop_reason: Option, pub stop_sequence: Option, #[serde(skip_serializing_if = "Option::is_none")] @@ -20,3 +18,61 @@ pub struct AnthropicMessagesResponse { #[serde(flatten)] pub extra: Map, } + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn response( + stop_reason: Option<&str>, + stop_sequence: Option<&str>, + usage: Option, + container: Option, + ) -> AnthropicMessagesResponse { + AnthropicMessagesResponse { + id: "msg_1".to_string(), + message_type: "message".to_string(), + role: "assistant".to_string(), + model: "claude".to_string(), + content: vec![], + stop_reason: stop_reason.map(str::to_string), + stop_sequence: stop_sequence.map(str::to_string), + usage, + container, + extra: Map::new(), + } + } + + #[rstest] + #[case::turn_in_progress(None, None, json!(null), json!(null))] + #[case::ended_on_end_turn(Some("end_turn"), None, json!("end_turn"), json!(null))] + #[case::ended_on_stop_sequence(Some("stop_sequence"), Some("###"), json!("stop_sequence"), json!("###"))] + fn stop_fields_are_always_serialized( + #[case] stop_reason: Option<&str>, + #[case] stop_sequence: Option<&str>, + #[case] expected_reason: Value, + #[case] expected_sequence: Value, + ) { + let body: Value = serde_json::to_value(response(stop_reason, stop_sequence, None, None)) + .expect("serializable"); + assert_eq!(body.get("stop_reason"), Some(&expected_reason)); + assert_eq!(body.get("stop_sequence"), Some(&expected_sequence)); + } + + #[rstest] + #[case::absent(None, None)] + #[case::present(Some(json!({"input_tokens": 1})), Some(json!({"id": "c_1"})))] + fn usage_and_container_are_omitted_only_when_none( + #[case] usage: Option, + #[case] container: Option, + ) { + let body: Value = + serde_json::to_value(response(None, None, usage.clone(), container.clone())) + .expect("serializable"); + assert_eq!(body.get("usage").cloned(), usage); + assert_eq!(body.get("container").cloned(), container); + } +} diff --git a/litellm-rust/crates/types/src/utils.rs b/litellm-rust/crates/types/src/utils.rs index 7f0c18f9f2c..5ca56ec9e49 100644 --- a/litellm-rust/crates/types/src/utils.rs +++ b/litellm-rust/crates/types/src/utils.rs @@ -3,6 +3,21 @@ use serde_json::{Map, Value}; use crate::llms::openai::{ChatCompletionThinkingBlock, ChatCompletionToolCallChunk}; +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +pub struct ProviderSpecificHeader { + #[serde(default)] + pub custom_llm_provider: String, + #[serde(default)] + pub extra_headers: Map, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +pub enum ProviderSpecificHeaders { + One(ProviderSpecificHeader), + Many(Vec), +} + /// OpenAI `usage`, including the `prompt_tokens_details` split LiteLLM's Python /// path reports so cost tracking sees the same numbers on either path. #[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] diff --git a/litellm/rust_bridge/messages/route_host.py b/litellm/rust_bridge/messages/route_host.py index beef0f81eca..d49d7b75a6f 100644 --- a/litellm/rust_bridge/messages/route_host.py +++ b/litellm/rust_bridge/messages/route_host.py @@ -1,12 +1,50 @@ from __future__ import annotations -from collections.abc import Mapping -from typing import cast # noqa: TID251 # narrows the normalized native payload to the public TypedDict +from collections.abc import Mapping, Sequence +from dataclasses import asdict, dataclass +from typing import Final, cast # noqa: TID251 # narrows the normalized native payload to the public TypedDict +from pydantic import TypeAdapter, ValidationError + +import litellm +from litellm.litellm_core_utils.core_helpers import normalize_drop_params +from litellm.llms.anthropic.experimental_pass_through.utils import is_reasoning_auto_summary_enabled from litellm.rust_bridge import failures from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest from litellm.types.llms.anthropic_messages.anthropic_response import AnthropicMessagesResponse +_DROP_PATHS: Final = TypeAdapter(list[object]) + + +@dataclass(frozen=True, slots=True) +class EffortTiers: + minimal: bool + low: bool + medium: bool + high: bool + xhigh: bool + max: bool + + +@dataclass(frozen=True, slots=True) +class ModelCapabilities: + supports_reasoning: bool + supports_adaptive_thinking: bool + thinking_always_on: bool + supports_legacy_thinking: bool + supports_output_config: bool + supports_sampling_params: bool + supports_speed: bool + effort_tiers: EffortTiers + + +@dataclass(frozen=True, slots=True) +class MessagesShaping: + capabilities: ModelCapabilities + drop_params: bool + reasoning_auto_summary: bool + additional_drop_params: Sequence[str] + def response(value: Mapping[str, object]) -> AnthropicMessagesResponse: return cast( # cast-ok: AnthropicMessagesResponse is a TypedDict over the normalized native payload @@ -20,4 +58,72 @@ def arguments(request: LiteLLMMessagesRequest) -> Mapping[str, object]: def map_failure(error: Exception, request: LiteLLMMessagesRequest, request_provider: str) -> Exception: + if getattr(error, "messages_request_error", False): + return litellm.BadRequestError( + message=str(error), + model=request.model.removeprefix(f"{request_provider}/"), + llm_provider=request_provider, + ) return failures.map_native_failure(error, request.model, request_provider, arguments(request), request.api_base) + + +def _resolved_provider(model: str, custom_llm_provider: str | None) -> tuple[str, str]: + try: + resolved_model, provider, _, _ = litellm.get_llm_provider(model=model, custom_llm_provider=custom_llm_provider) + except Exception: # noqa: BLE001 # an unroutable model still shapes as a bare Anthropic id + return model, custom_llm_provider or "anthropic" + return resolved_model, provider + + +def model_capabilities(model: str, custom_llm_provider: str | None) -> ModelCapabilities: + from litellm.llms.anthropic.chat.transformation import AnthropicConfig + from litellm.llms.anthropic.common_utils import AnthropicModelInfo + + resolved_model, provider = _resolved_provider(model, custom_llm_provider) + + def supports(flag: str) -> bool: + return AnthropicModelInfo._supports_model_capability(model, flag, provider) # pyright: ignore[reportPrivateUsage] # same probes the Python transform runs; forking them would drift + + def tier(level: str) -> bool: + return AnthropicConfig._supports_effort_level(model, level, provider) # pyright: ignore[reportPrivateUsage] # same probe the Python transform runs + + return ModelCapabilities( + supports_reasoning=supports("supports_reasoning"), + supports_adaptive_thinking=supports("supports_adaptive_thinking"), + thinking_always_on=supports("thinking_always_on"), + supports_legacy_thinking=supports("supports_legacy_thinking"), + supports_output_config=supports("supports_output_config"), + supports_sampling_params=AnthropicModelInfo._supports_sampling_params(resolved_model), # pyright: ignore[reportPrivateUsage] # same gate the handler applies + supports_speed=AnthropicConfig._model_supports_speed_param(resolved_model, provider), # pyright: ignore[reportPrivateUsage] # same gate the handler applies + effort_tiers=EffortTiers( + minimal=tier("minimal"), + low=tier("low"), + medium=tier("medium"), + high=tier("high"), + xhigh=tier("xhigh"), + max=tier("max"), + ), + ) + + +def _drop_params(kwargs: Mapping[str, object]) -> bool: + return bool(litellm.drop_params) or normalize_drop_params(kwargs.get("drop_params")) is True + + +def _additional_drop_params(kwargs: Mapping[str, object]) -> tuple[str, ...]: + try: + configured: Final = _DROP_PATHS.validate_python(kwargs.get("additional_drop_params")) + except ValidationError: + return () + return tuple(path for path in configured if isinstance(path, str)) + + +def shaping(model: str, custom_llm_provider: str | None, kwargs: Mapping[str, object]) -> dict[str, object]: + return asdict( + MessagesShaping( + capabilities=model_capabilities(model, custom_llm_provider), + drop_params=_drop_params(kwargs), + reasoning_auto_summary=is_reasoning_auto_summary_enabled(), + additional_drop_params=_additional_drop_params(kwargs), + ) + ) diff --git a/tests/test_litellm/rust_bridge/AGENTS.md b/tests/test_litellm/rust_bridge/AGENTS.md index 351bd582ce7..994d24112e8 100644 --- a/tests/test_litellm/rust_bridge/AGENTS.md +++ b/tests/test_litellm/rust_bridge/AGENTS.md @@ -5,3 +5,5 @@ Test what each side of the bridge does, not the rollout policy that picks a side Call each path directly with an explicit decision instead. The Python path is the implementation the dispatcher falls back to, e.g. `litellm.ocr.main.ocr`. The Rust path is the native binding, e.g. `NATIVE_OCR.load()` from `litellm/rust_bridge/ocr/entrypoints.py`, called with the request, args and kwargs that dispatch would hand it. When the native side reads a policy-derived setting such as `settings.secret_manager().native`, pin that field in the test instead of deriving it from the catalog. `ocr/test_secrets.py` shows the pattern Rollout policy itself, meaning which rule matches and what `LITELLM_RUST` changes, belongs in `test_catalog.py`, `test_configuration.py` and `test_dispatch.py`, tested against rules the test builds rather than the shipped `catalog.RULES` + +Before adding a test here, ask whether it checks something Rust cannot. A `route_host.py` module is the Python half of a native route: it projects Python-only state (the cost map, `litellm.*` settings, request kwargs) into the plain values the Rust side consumes, and maps native failures back onto public exceptions. Those projections are what belongs here, because a wrong key or an ignored provider prefix ships the wrong value to Rust and no Rust test sees it. `messages/test_route_host.py` shows the shape. Behavior that lives in Rust (a request transform given its inputs, header assembly, stream relay) is tested in the crate, and the route end to end is tested against a recording server in `tests/test_litellm_rust/`. A test that only re-checks a Python helper the route host happens to call is a duplicate of that helper's own test and should not be added diff --git a/tests/test_litellm/rust_bridge/messages/test_route_host.py b/tests/test_litellm/rust_bridge/messages/test_route_host.py new file mode 100644 index 00000000000..f47333a45d9 --- /dev/null +++ b/tests/test_litellm/rust_bridge/messages/test_route_host.py @@ -0,0 +1,112 @@ +from dataclasses import astuple +from typing import Final + +import pytest + +import litellm +from litellm.rust_bridge.messages import route_host + +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + + +def _flag_model(monkeypatch: pytest.MonkeyPatch, name: str, **flags: bool) -> None: + monkeypatch.setitem( + litellm.model_cost, + name, + { + "litellm_provider": "anthropic", + "mode": "chat", + "input_cost_per_token": 0, + "output_cost_per_token": 0, + **flags, + }, + ) + + +def test_capabilities_come_from_the_model_map_under_the_callers_provider(monkeypatch: pytest.MonkeyPatch) -> None: + _flag_model( + monkeypatch, + "claude-test-adaptive", + supports_reasoning=True, + supports_adaptive_thinking=True, + supports_output_config=True, + supports_xhigh_reasoning_effort=True, + supports_sampling_params=False, + ) + + capabilities: Final = route_host.model_capabilities("anthropic/claude-test-adaptive", None) + + assert capabilities.supports_adaptive_thinking + assert capabilities.supports_output_config + assert not capabilities.supports_legacy_thinking + assert not capabilities.supports_sampling_params + assert capabilities.effort_tiers.xhigh + assert not capabilities.effort_tiers.max + + +def test_unmapped_model_keeps_sampling_params_and_no_reasoning_features() -> None: + capabilities: Final = route_host.model_capabilities("anthropic/not-a-real-model", None) + + assert capabilities.supports_sampling_params + assert not capabilities.supports_reasoning + assert not capabilities.supports_adaptive_thinking + assert not any(astuple(capabilities.effort_tiers)) + + +@pytest.mark.parametrize( + ("global_flag", "kwargs", "expected"), + [ + (False, {}, False), + (True, {}, True), + (False, {"drop_params": "true"}, True), + (False, {"drop_params": "nonsense"}, False), + (False, {"drop_params": False}, False), + ], +) +def test_drop_params_merges_the_global_flag_with_the_request( + monkeypatch: pytest.MonkeyPatch, global_flag: bool, kwargs: dict[str, object], expected: bool +) -> None: + monkeypatch.setattr(litellm, "drop_params", global_flag) + + assert route_host.shaping("anthropic/not-a-real-model", None, kwargs)["drop_params"] is expected + + +@pytest.mark.parametrize( + ("configured", "expected"), + [ + (["tools[*].input_examples", 3, "metadata.user_id"], ("tools[*].input_examples", "metadata.user_id")), + ("tools", ()), + (None, ()), + ], +) +def test_additional_drop_params_keep_only_string_paths(configured: object, expected: tuple[str, ...]) -> None: + shaping: Final = route_host.shaping("anthropic/not-a-real-model", None, {"additional_drop_params": configured}) + + assert shaping["additional_drop_params"] == expected + + +def test_native_request_rejections_map_to_the_public_400() -> None: + from types import MappingProxyType + + from litellm.rust_bridge.messages.entrypoints import LiteLLMMessagesRequest + + request: Final = LiteLLMMessagesRequest( + model="anthropic/claude-sonnet-5", + messages=(), + max_tokens=8, + stream=None, + api_key=None, + api_base=None, + custom_llm_provider=None, + kwargs=MappingProxyType({}), + ) + rejected: Final = ValueError("claude-sonnet-5 does not support top_k=5") + rejected.messages_request_error = True # pyright: ignore[reportAttributeAccessIssue] # marker the native host sets + + mapped: Final = route_host.map_failure(rejected, request, "anthropic") + + assert isinstance(mapped, litellm.BadRequestError) + assert mapped.status_code == 400 + assert "does not support top_k=5" in mapped.message + assert mapped.model == "claude-sonnet-5" + assert not isinstance(route_host.map_failure(ValueError("plain"), request, "anthropic"), litellm.BadRequestError) diff --git a/tests/test_litellm/rust_bridge/messages/test_secrets.py b/tests/test_litellm/rust_bridge/messages/test_secrets.py new file mode 100644 index 00000000000..cf37ed0830b --- /dev/null +++ b/tests/test_litellm/rust_bridge/messages/test_secrets.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from collections.abc import Awaitable, Mapping +from dataclasses import replace +from types import MappingProxyType +from typing import Final, Protocol, cast # noqa: TID251 # narrows the parametrized path to its protocol + +import httpx +import pytest + +import litellm +from litellm.integrations.custom_secret_manager import CustomSecretManager +from litellm.llms.anthropic.experimental_pass_through.messages.handler import anthropic_messages +from litellm.rust_bridge import settings +from litellm.rust_bridge.messages.entrypoints import NATIVE_AMESSAGES, NATIVE_MESSAGES, LiteLLMMessagesRequest +from litellm.types.secret_managers.main import KeyManagementSettings, KeyManagementSystem +from tests.test_litellm_rust.support.recording_server import ResponseSpec, recording_service +from tests.test_litellm_rust.support.requests import MESSAGES, MESSAGES_MODEL, MESSAGES_RESPONSE + +pytest.importorskip("litellm.rust_bridge._native") + +pytestmark = pytest.mark.usefixtures("local_model_cost_map") + + +class Messages(Protocol): + def __call__(self) -> Awaitable[object]: ... + + +class _ManagedSecrets(CustomSecretManager): + def __init__(self, values: Mapping[str, str]) -> None: + super().__init__(secret_manager_name="rust_bridge_messages_test") + self.values: Final = values + + async def async_read_secret( + self, + secret_name: str, + optional_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> str | None: + raise AssertionError("get_secret reads custom managers synchronously") + + def sync_read_secret( + self, + secret_name: str, + optional_params: dict[str, object] | None = None, + timeout: float | httpx.Timeout | None = None, + ) -> str | None: + return self.values.get(secret_name) + + +def _native_request() -> LiteLLMMessagesRequest: + return LiteLLMMessagesRequest( + model=MESSAGES_MODEL, + messages=MESSAGES, + max_tokens=8, + stream=None, + api_key=None, + api_base=None, + custom_llm_provider=None, + kwargs=MappingProxyType({}), + ) + + +def _public_kwargs() -> dict[str, object]: + return {"model": MESSAGES_MODEL, "messages": [dict(message) for message in MESSAGES], "max_tokens": 8} + + +async def _python_messages() -> object: + return await anthropic_messages(**_public_kwargs()) + + +async def _rust_messages() -> object: + route: Final = NATIVE_MESSAGES.load() + assert route is not None + return route(_native_request(), (), _public_kwargs()) + + +async def _rust_amessages() -> object: + route: Final = NATIVE_AMESSAGES.load() + assert route is not None + return await route(_native_request(), (), _public_kwargs()) + + +@pytest.fixture( + params=(_python_messages, _rust_messages, _rust_amessages), ids=("python-async", "rust-sync", "rust-async") +) +def messages(request: pytest.FixtureRequest) -> Messages: + return cast(Messages, request.param) + + +async def test_secret_manager_supplies_the_anthropic_key_and_base( + monkeypatch: pytest.MonkeyPatch, messages: Messages +) -> None: + for name in ("ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_API_BASE", "ANTHROPIC_BASE_URL"): + monkeypatch.delenv(name, raising=False) + with recording_service() as server: + server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + monkeypatch.setattr( + litellm, + "secret_manager_client", + _ManagedSecrets({"ANTHROPIC_API_KEY": "vault-key", "ANTHROPIC_BASE_URL": server.base_url}), + ) + monkeypatch.setattr(litellm, "_key_management_system", KeyManagementSystem.CUSTOM) + monkeypatch.setattr(litellm, "_key_management_settings", KeyManagementSettings(access_mode="read_only")) + configured: Final = settings.secret_manager + monkeypatch.setattr(settings, "secret_manager", lambda: replace(configured(), native=True)) + + await messages() + + assert len(server.requests) == 1 + assert server.requests[0].headers["x-api-key"] == "vault-key" diff --git a/tests/test_litellm_rust/messages/test_request_shaping.py b/tests/test_litellm_rust/messages/test_request_shaping.py new file mode 100644 index 00000000000..f885fda5f42 --- /dev/null +++ b/tests/test_litellm_rust/messages/test_request_shaping.py @@ -0,0 +1,237 @@ +"""The native Messages route shapes the wire request the way the Python handler does. + +Model capability expectations come from model_prices_and_context_window.json (Claude Sonnet 5 is an +adaptive-thinking model without sampling params; Claude Haiku 4.5 is a legacy-thinking model), read at +2026-09-24; the cost map is LiteLLM's own file. +""" + +from collections.abc import Iterator +from typing import Final + +import pytest + +import litellm +from litellm.rust_bridge import catalog +from litellm.rust_bridge.catalog import Route, RouteRule +from litellm.rust_bridge.configuration import Rollout +from tests.test_litellm_rust.support.isolation import rebound +from tests.test_litellm_rust.support.recording_server import RecordingServer, ResponseSpec +from tests.test_litellm_rust.support.requests import MESSAGES, MESSAGES_RESPONSE + +pytestmark = pytest.mark.requires_rust_extension + +ADAPTIVE_MODEL: Final = "anthropic/claude-sonnet-5" +LEGACY_THINKING_MODEL: Final = "anthropic/claude-haiku-4-5" + + +@pytest.fixture(autouse=True) +def opt_messages_into_rust() -> Iterator[None]: + with rebound(catalog, "RULES", (RouteRule(Route.MESSAGES, Rollout.RUST_OPT_IN), *catalog.RULES)): + yield + + +@pytest.fixture +def messages_server(recording_server: RecordingServer) -> RecordingServer: + recording_server.default_response = ResponseSpec(body=MESSAGES_RESPONSE) + return recording_server + + +def arguments(server: RecordingServer, **kwargs: object) -> dict[str, object]: + return { + "model": ADAPTIVE_MODEL, + "messages": [dict(message) for message in MESSAGES], + "max_tokens": 8192, + "api_key": "test-key", + "api_base": server.base_url, + **kwargs, + } + + +def sent(server: RecordingServer) -> tuple[dict[str, object], dict[str, str]]: + assert len(server.requests) == 1 + request: Final = server.requests[0] + assert not request.headers.get("user-agent", "").startswith("python-httpx") + assert isinstance(request.body, dict) + return request.body, request.headers + + +@pytest.mark.asyncio +async def test_reasoning_effort_becomes_adaptive_thinking_and_effort_on_the_wire( + messages_server: RecordingServer, +) -> None: + await litellm.anthropic.messages.acreate(**arguments(messages_server, reasoning_effort="high")) + + body, _ = sent(messages_server) + assert "reasoning_effort" not in body + assert body["thinking"] == {"type": "adaptive", "display": "summarized"} + assert body["output_config"] == {"effort": "high"} + + +@pytest.mark.asyncio +async def test_claude_code_adaptive_payload_is_downgraded_to_a_capped_budget_for_a_legacy_model( + messages_server: RecordingServer, +) -> None: + await litellm.anthropic.messages.acreate( + **arguments( + messages_server, + model=LEGACY_THINKING_MODEL, + max_tokens=3000, + thinking={"type": "adaptive"}, + output_config={"effort": "high"}, + temperature=0, + ) + ) + + body, _ = sent(messages_server) + assert body["thinking"] == {"type": "enabled", "budget_tokens": 2999} + assert "output_config" not in body + assert "temperature" not in body + + +@pytest.mark.asyncio +async def test_removed_sampling_params_are_dropped_under_drop_params(messages_server: RecordingServer) -> None: + await litellm.anthropic.messages.acreate( + **arguments(messages_server, temperature=0.2, top_p=0.9, top_k=5, drop_params=True) + ) + + body, _ = sent(messages_server) + assert not {"temperature", "top_p", "top_k"} & body.keys() + + +@pytest.mark.asyncio +async def test_removed_sampling_params_are_rejected_without_drop_params(messages_server: RecordingServer) -> None: + messages_server.expected_requests = 0 + + with pytest.raises(litellm.BadRequestError, match="does not support top_k=5"): + await litellm.anthropic.messages.acreate(**arguments(messages_server, top_k=5)) + + assert messages_server.requests == [] + + +@pytest.mark.asyncio +async def test_replayed_history_is_sanitized_before_it_reaches_the_provider( + messages_server: RecordingServer, +) -> None: + history: Final = [ + {"role": "user", "content": "run it"}, + { + "role": "assistant", + "content": [ + {"type": "text", "text": ""}, + { + "type": "tool_use", + "id": "functions.Bash:0", + "name": "Bash", + "input": {}, + "provider_specific_fields": {"x": 1}, + }, + ], + }, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "functions.Bash:0", "content": "ok"}]}, + ] + + await litellm.anthropic.messages.acreate(**arguments(messages_server, messages=history)) + + body, _ = sent(messages_server) + assert body["messages"] == [ + {"role": "user", "content": "run it"}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "functions_Bash_0", "name": "Bash", "input": {}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "functions_Bash_0", "content": "ok"}]}, + ] + + +@pytest.mark.asyncio +async def test_feature_betas_merge_into_the_forwarded_beta_header(messages_server: RecordingServer) -> None: + await litellm.anthropic.messages.acreate( + **arguments( + messages_server, + output_format={"type": "json_schema", "schema": {"type": "object"}}, + extra_headers={"anthropic-beta": "web-search-2025-03-05"}, + ) + ) + + _, headers = sent(messages_server) + assert headers["anthropic-beta"] == "structured-outputs-2025-11-13,web-search-2025-03-05" + + +@pytest.mark.asyncio +async def test_oauth_token_authenticates_as_a_bearer_with_the_oauth_beta(messages_server: RecordingServer) -> None: + await litellm.anthropic.messages.acreate(**arguments(messages_server, api_key="sk-ant-oat01-token")) + + _, headers = sent(messages_server) + assert "x-api-key" not in headers + assert headers["authorization"] == "Bearer sk-ant-oat01-token" + assert headers["anthropic-beta"] == "oauth-2025-04-20" + assert headers["anthropic-dangerous-direct-browser-access"] == "true" + + +@pytest.mark.asyncio +async def test_metadata_is_reduced_to_the_fields_anthropic_accepts(messages_server: RecordingServer) -> None: + await litellm.anthropic.messages.acreate( + **arguments(messages_server, metadata={"user_id": "u-1", "trace_id": "internal"}) + ) + + body, _ = sent(messages_server) + assert body["metadata"] == {"user_id": "u-1"} + + +@pytest.mark.asyncio +async def test_additional_drop_params_remove_nested_fields_from_the_wire(messages_server: RecordingServer) -> None: + tools: Final = [{"name": "lookup", "input_schema": {"type": "object"}, "input_examples": [{"q": "x"}]}] + + await litellm.anthropic.messages.acreate( + **arguments(messages_server, tools=tools, additional_drop_params=["tools[*].input_examples"]) + ) + + body, _ = sent(messages_server) + assert body["tools"] == [{"name": "lookup", "input_schema": {"type": "object"}}] + + +@pytest.mark.asyncio +async def test_provider_specific_headers_scoped_to_anthropic_reach_the_wire(messages_server: RecordingServer) -> None: + await litellm.anthropic.messages.acreate( + **arguments( + messages_server, + provider_specific_header=[ + {"custom_llm_provider": "anthropic, azure_ai", "extra_headers": {"x-scoped": "yes"}}, + {"custom_llm_provider": "openai", "extra_headers": {"x-other": "no"}}, + ], + ) + ) + + _, headers = sent(messages_server) + assert headers["x-scoped"] == "yes" + assert "x-other" not in headers + + +@pytest.mark.asyncio +async def test_scoped_headers_override_extra_headers_which_override_forwarded_headers( + messages_server: RecordingServer, +) -> None: + await litellm.anthropic.messages.acreate( + **arguments( + messages_server, + headers={"x-priority": "forwarded", "x-forwarded-only": "kept"}, + extra_headers={"x-priority": "extra", "x-extra-only": "kept"}, + provider_specific_header={"custom_llm_provider": "anthropic", "extra_headers": {"x-priority": "scoped"}}, + ) + ) + + _, headers = sent(messages_server) + assert {name: headers.get(name) for name in ("x-priority", "x-forwarded-only", "x-extra-only")} == { + "x-priority": "scoped", + "x-forwarded-only": "kept", + "x-extra-only": "kept", + } + + +@pytest.mark.asyncio +async def test_non_string_metadata_user_id_is_rejected_before_the_provider_call( + messages_server: RecordingServer, +) -> None: + messages_server.expected_requests = 0 + + with pytest.raises(litellm.BadRequestError, match=r"metadata\.user_id must be a string"): + await litellm.anthropic.messages.acreate(**arguments(messages_server, metadata={"user_id": 123})) + + assert messages_server.requests == []