diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4b741c993e4..1c4e6553b66 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4055,6 +4055,7 @@ dependencies = [ name = "litellm-inference-chat" version = "0.1.0" dependencies = [ + "base64 0.22.1", "bytes", "futures-util", "litellm-auth", @@ -4235,11 +4236,13 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_with", + "sha2 0.10.9", "strum", "thiserror 2.0.19", "time", "tokio", "url", + "uuid", ] [[package]] diff --git a/litellm-rust/crates/gateway-inference/src/chat_completions.rs b/litellm-rust/crates/gateway-inference/src/chat_completions.rs index 6f4442dd053..84abff5fe70 100644 --- a/litellm-rust/crates/gateway-inference/src/chat_completions.rs +++ b/litellm-rust/crates/gateway-inference/src/chat_completions.rs @@ -146,6 +146,8 @@ async fn handle( custom_llm_provider: deployment.custom_llm_provider.clone(), extra_headers: extra_headers.clone(), timeout: deployment.timeout, + drop_params: Some(drop_params(gateway, deployment)), + model_info: gateway.model_info(), }; if stream { return translated_stream(route, call, name).await; @@ -197,6 +199,17 @@ fn provider_params(deployment: &Deployment) -> Map { provider_params } +/// `litellm.drop_params` or the deployment's own: either one set drops what the provider +/// cannot take, as Python's `get_optional_params` reads them. +fn drop_params(gateway: &Gateway, deployment: &Deployment) -> bool { + gateway.drop_params() + || deployment + .litellm_params + .drop_params + .as_ref() + .is_some_and(|flag| flag.is_set()) +} + async fn openai( gateway: &Gateway, deployment: &Deployment, @@ -213,12 +226,7 @@ async fn openai( extra_headers, timeout: deployment.timeout, litellm_params: provider_params(deployment), - drop_params: gateway.drop_params() - || deployment - .litellm_params - .drop_params - .as_ref() - .is_some_and(|flag| flag.is_set()), + drop_params: drop_params(gateway, deployment), model_info: gateway.model_info(), }, None, diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 65aa2d72779..010e92cfbe1 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -18,7 +18,10 @@ mod responses; use std::sync::{Arc, OnceLock}; use axum::{Router, routing::post}; -use litellm_http::{ClientVariant, HttpClientConfig, media::UrlPolicy}; +use litellm_http::{ + ClientVariant, HttpClientConfig, + media::{MediaFetcher, UrlPolicy}, +}; use litellm_inference::resources::CoreResources; use litellm_inference_chat::ChatCompletionsRoute; use litellm_inference_messages::MessagesRoute; @@ -138,7 +141,12 @@ impl Gateway { provider.clone(), auth.clone(), secrets.clone(), - ), + ) + .with_media(MediaFetcher::new( + &resources.pool, + &http, + UrlPolicy::default(), + )?), messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone()), responses: ResponsesRoute::new(provider, auth.clone(), secrets.clone()), ocr: OcrRoute::new(OcrClient::new( diff --git a/litellm-rust/crates/gateway-inference/tests/routes.rs b/litellm-rust/crates/gateway-inference/tests/routes.rs index 73925399e38..7452306338e 100644 --- a/litellm-rust/crates/gateway-inference/tests/routes.rs +++ b/litellm-rust/crates/gateway-inference/tests/routes.rs @@ -105,6 +105,8 @@ async fn chat_errors_come_from_core( custom_llm_provider: None, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, }, &(), None, diff --git a/litellm-rust/crates/gateway/src/hooks.rs b/litellm-rust/crates/gateway/src/hooks.rs index 6c393816cbf..9f315cd3f78 100644 --- a/litellm-rust/crates/gateway/src/hooks.rs +++ b/litellm-rust/crates/gateway/src/hooks.rs @@ -1,7 +1,7 @@ //! Builds the hooks and observers inference requests run through, and the per-deployment prices //! observers that read cost need. -use std::{collections::HashSet, sync::Arc}; +use std::{borrow::Cow, collections::HashSet, sync::Arc}; use litellm_config::{Config, Model}; use litellm_gateway_hooks::{ @@ -30,19 +30,19 @@ pub async fn install( hooks: Vec>, observers: Vec>, ) -> Result<(), Error> { - let serves_azure = config.model_list.iter().any(is_azure); - if hooks.is_empty() && observers.is_empty() && !serves_azure { + let reads_model_info = config.model_list.iter().any(reads_model_info); + if hooks.is_empty() && observers.is_empty() && !reads_model_info { return Ok(()); } let needs_cost = observers .iter() .any(|observer| observer.reads().contains(Facts::COST)) || hooks.iter().any(|hook| hook.reads().contains(Facts::COST)); - let catalog = match needs_cost || serves_azure { + let catalog = match needs_cost || reads_model_info { true => load_catalog(inference).await.map(Arc::new), false => None, }; - if let Some(catalog) = catalog.as_ref().filter(|_| serves_azure) { + if let Some(catalog) = catalog.as_ref().filter(|_| reads_model_info) { inference.install_model_info(Arc::new(CostMap::new(catalog.clone()))); } if hooks.is_empty() && observers.is_empty() { @@ -204,7 +204,19 @@ fn is_azure(model: &Model) -> bool { } } -/// The model cost map as the capabilities Azure's param rules read. +/// Whether the deployment's chat param rules read model capabilities from the cost map: +/// Azure, and the Anthropic and Bedrock chat translations. +fn reads_model_info(model: &Model) -> bool { + is_azure(model) + || litellm_inference::provider::resolve_llm_provider( + &model.litellm_params.model, + model.litellm_params.custom_llm_provider.as_deref(), + "chat completions", + ) + .is_ok_and(|resolved| matches!(<&str>::from(resolved.provider), "anthropic" | "bedrock")) +} + +/// The model cost map as the capabilities the chat param rules read. struct CostMap { catalog: Arc, keys: HashSet, @@ -222,8 +234,10 @@ impl CostMap { } impl ModelInfo for CostMap { - fn entry(&self, model: &str) -> Option<&Map> { - self.catalog.lookup(model).map(|found| found.entry.fields()) + fn entry(&self, model: &str) -> Option>> { + self.catalog + .lookup(model) + .map(|found| Cow::Borrowed(found.entry.fields())) } fn declares(&self, key: &str) -> bool { diff --git a/litellm-rust/crates/gateway/tests/chat_translation.rs b/litellm-rust/crates/gateway/tests/chat_translation.rs new file mode 100644 index 00000000000..aa0c1da8f6f --- /dev/null +++ b/litellm-rust/crates/gateway/tests/chat_translation.rs @@ -0,0 +1,722 @@ +//! Chat completions for deployments the gateway translates (Anthropic, Bedrock Converse): the +//! OpenAI request widened to tools, images and the provider's own limits, the provider body +//! the upstream sees, and the OpenAI answer the client gets back, streamed and not. + +use std::{ + io::Write, + sync::{Arc, Mutex, Once}, +}; + +use axum::{ + Router, + body::{Body, Bytes, to_bytes}, + http::{Request, header::CONTENT_TYPE}, + response::IntoResponse, +}; +use litellm_config::Config; +use litellm_framer::aws_event_stream::{AwsEventStreamCodec, Header, HeaderValue, Message}; +use rstest::rstest; +use serde_json::{Value, json}; +use tokio::net::TcpListener; +use tokio_util::codec::Encoder; +use tower::ServiceExt; + +const ANTHROPIC_MODEL: &str = "claude-x"; +const BEDROCK_MODEL: &str = "anthropic.claude-x-v1:0"; +/// A one-pixel PNG. +const PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + +/// The gateway reads its cost map from this variable; every test sets it before building a +/// gateway, so no test reads the environment while it changes. +fn use_local_cost_map() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + let capabilities = json!({ + "mode": "chat", + "max_tokens": 8192, + "max_output_tokens": 8192, + "supports_tool_choice": true, + "supports_function_calling": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + }); + let entry = |provider: &str| { + let mut entry = capabilities.clone(); + entry["litellm_provider"] = json!(provider); + entry + }; + let map = json!({ + ANTHROPIC_MODEL: entry("anthropic"), + BEDROCK_MODEL: entry("bedrock_converse"), + }); + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(map.to_string().as_bytes()).unwrap(); + let (_, path) = file.keep().unwrap(); + // SAFETY: set once, before any test builds a gateway or spawns a thread that reads the + // environment. + unsafe { std::env::set_var("LITELLM_MODEL_COST_MAP_PATH", path) }; + }); +} + +fn anthropic_events(events: &[Value]) -> String { + events + .iter() + .map(|event| { + format!( + "event: {}\ndata: {event}\n\n", + event["type"].as_str().unwrap() + ) + }) + .collect() +} + +fn anthropic_tool_message() -> Value { + json!({ + "id": "msg_1", "type": "message", "role": "assistant", "model": ANTHROPIC_MODEL, + "content": [ + {"type": "text", "text": "Checking."}, + {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}} + ], + "stop_reason": "tool_use", "stop_sequence": null, + "usage": {"input_tokens": 10, "output_tokens": 5} + }) +} + +/// The answer to a `response_format` request, given as the tool it became. +fn anthropic_json_message() -> Value { + json!({ + "id": "msg_3", "type": "message", "role": "assistant", "model": ANTHROPIC_MODEL, + "content": [{"type": "tool_use", "id": "toolu_2", "name": "json_tool_call", "input": {"city": "Paris"}}], + "stop_reason": "tool_use", "stop_sequence": null, + "usage": {"input_tokens": 10, "output_tokens": 5} + }) +} + +fn anthropic_text_message() -> Value { + json!({ + "id": "msg_2", "type": "message", "role": "assistant", "model": ANTHROPIC_MODEL, + "content": [{"type": "text", "text": "Sunny."}], + "stop_reason": "end_turn", "stop_sequence": null, + "usage": {"input_tokens": 10, "output_tokens": 2} + }) +} + +fn anthropic_tool_stream() -> String { + anthropic_events(&[ + json!({"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": ANTHROPIC_MODEL, "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 10, "output_tokens": 1}}}), + json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {}}}), + json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"city\": "}}), + json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "\"Paris\"}"}}), + json!({"type": "content_block_stop", "index": 0}), + json!({"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, "usage": {"output_tokens": 5}}), + json!({"type": "message_stop"}), + ]) +} + +fn converse_tool_response() -> Value { + json!({ + "output": {"message": {"role": "assistant", "content": [ + {"text": "Checking."}, + {"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather", "input": {"city": "Paris"}}} + ]}}, + "stopReason": "tool_use", + "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15} + }) +} + +fn converse_text_response() -> Value { + json!({ + "output": {"message": {"role": "assistant", "content": [{"text": "Sunny."}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 2, "totalTokens": 12} + }) +} + +fn bedrock_frame(name: &str, payload: Value, wire: &mut bytes::BytesMut) { + let message = Message::new(Bytes::from(payload.to_string())) + .add_header(Header::new( + ":message-type", + HeaderValue::String("event".to_owned().into()), + )) + .add_header(Header::new( + ":event-type", + HeaderValue::String(name.to_owned().into()), + )); + AwsEventStreamCodec.encode(message, wire).unwrap(); +} + +fn converse_tool_stream() -> Bytes { + let mut wire = bytes::BytesMut::new(); + for (name, payload) in [ + ("messageStart", json!({"role": "assistant"})), + ( + "contentBlockStart", + json!({"contentBlockIndex": 0, "start": {"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather"}}}), + ), + ( + "contentBlockDelta", + json!({"contentBlockIndex": 0, "delta": {"toolUse": {"input": "{\"city\": "}}}), + ), + ( + "contentBlockDelta", + json!({"contentBlockIndex": 0, "delta": {"toolUse": {"input": "\"Paris\"}"}}}), + ), + ("contentBlockStop", json!({"contentBlockIndex": 0})), + ("messageStop", json!({"stopReason": "tool_use"})), + ( + "metadata", + json!({"usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, "metrics": {"latencyMs": 9}}), + ), + ] { + bedrock_frame(name, payload, &mut wire); + } + wire.freeze() +} + +#[derive(Clone, Debug)] +struct Sent { + path: String, + headers: Vec<(String, String)>, + body: Value, +} + +impl Sent { + fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + } +} + +type Calls = Arc>>; + +/// Answers each provider by path: a tool call while the conversation has no tool result yet, +/// the final answer once it does. +async fn upstream(calls: Calls) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = Router::new().fallback(move |request: Request| { + let calls = calls.clone(); + async move { + let (parts, body) = request.into_parts(); + let body: Value = + serde_json::from_slice(&to_bytes(body, 1 << 22).await.unwrap()).unwrap(); + let path = parts.uri.path().to_owned(); + calls.lock().unwrap().push(Sent { + path: path.clone(), + headers: parts + .headers + .iter() + .map(|(name, value)| { + ( + name.to_string(), + value.to_str().unwrap_or_default().to_owned(), + ) + }) + .collect(), + body: body.clone(), + }); + let answered = + body.to_string().contains("tool_result") || body.to_string().contains("toolResult"); + let json = [(CONTENT_TYPE, "application/json")]; + if path.ends_with("/converse-stream") { + return ( + [(CONTENT_TYPE, "application/vnd.amazon.eventstream")], + converse_tool_stream(), + ) + .into_response(); + } + if path.ends_with("/converse") { + let answer = match answered { + true => converse_text_response(), + false => converse_tool_response(), + }; + return (json, answer.to_string()).into_response(); + } + if body["stream"] == json!(true) { + return ( + [(CONTENT_TYPE, "text/event-stream")], + anthropic_tool_stream(), + ) + .into_response(); + } + let answer = match answered { + _ if body["tools"][0]["name"] == "json_tool_call" => anthropic_json_message(), + true => anthropic_text_message(), + false => anthropic_tool_message(), + }; + (json, answer.to_string()).into_response() + } + }); + tokio::spawn(async move { axum::serve(listener, app).await }); + format!("http://{address}") +} + +struct Harness { + app: Router, + calls: Calls, +} + +async fn harness() -> Harness { + use_local_cost_map(); + let calls = Calls::default(); + let upstream = upstream(calls.clone()).await; + let config = Config::from_yaml(&format!( + "model_list: + - model_name: claude + litellm_params: + model: anthropic/{ANTHROPIC_MODEL} + api_base: {upstream} + api_key: sk-anthropic + - model_name: bedrock + litellm_params: + model: bedrock/{BEDROCK_MODEL} + api_base: {upstream} + api_key: bedrock-token + - model_name: claude-dropping + litellm_params: + model: anthropic/{ANTHROPIC_MODEL} + api_base: {upstream} + api_key: sk-anthropic + drop_params: true +router_settings: + num_retries: 0 +general_settings: + master_key: sk-master +" + )) + .unwrap(); + let inference = litellm_gateway::build_inference(&config).unwrap(); + litellm_gateway::hooks::install(&inference, &config, Vec::new(), Vec::new()) + .await + .unwrap(); + let app = litellm_gateway::router(inference, &config, None, None); + Harness { app, calls } +} + +impl Harness { + async fn send(&self, body: Value) -> (u16, String) { + let request = Request::post("/v1/chat/completions") + .header("content-type", "application/json") + .header("authorization", "Bearer sk-master") + .body(Body::from(body.to_string())) + .unwrap(); + let response = self.app.clone().oneshot(request).await.unwrap(); + let status = response.status().as_u16(); + let body = to_bytes(response.into_body(), 1 << 22).await.unwrap(); + (status, String::from_utf8(body.to_vec()).unwrap()) + } + + async fn json(&self, body: Value) -> Value { + let (status, text) = self.send(body).await; + assert_eq!(status, 200, "{text}"); + serde_json::from_str(&text).unwrap() + } + + fn sent(&self) -> Vec { + self.calls.lock().unwrap().clone() + } +} + +fn weather_tool() -> Value { + json!({"type": "function", "function": { + "name": "get_weather", + "description": "The weather in a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]} + }}) +} + +fn question() -> Value { + json!({"role": "user", "content": "Weather in Paris?"}) +} + +/// The conversation after the model asked for the tool and the tool answered. +fn answered(call_id: &str) -> Value { + json!([ + question(), + {"role": "assistant", "content": null, "tool_calls": [ + {"id": call_id, "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}} + ]}, + {"role": "tool", "tool_call_id": call_id, "content": "sunny, 21C"} + ]) +} + +/// The streamed chunks' JSON and whether the stream ended with `[DONE]`. +fn frames(body: &str) -> (Vec, bool) { + let mut done = false; + let frames = body + .split("\n\n") + .filter_map(|frame| frame.strip_prefix("data: ")) + .filter(|data| { + let last = *data == "[DONE]"; + done |= last; + !last + }) + .map(|data| serde_json::from_str(data).unwrap()) + .collect(); + (frames, done) +} + +/// The tool calls the chunks stream, joined: (id, name, arguments) per index. +fn streamed_tool_calls(chunks: &[Value]) -> Vec<(String, String, String)> { + let mut calls: Vec<(String, String, String)> = Vec::new(); + for chunk in chunks { + let Some(deltas) = chunk["choices"][0]["delta"]["tool_calls"].as_array() else { + continue; + }; + for delta in deltas { + let index = delta["index"].as_u64().unwrap() as usize; + if calls.len() <= index { + calls.resize(index + 1, Default::default()); + } + let call = &mut calls[index]; + if let Some(id) = delta["id"].as_str() { + call.0 = id.to_owned(); + } + if let Some(name) = delta["function"]["name"].as_str() { + call.1 = name.to_owned(); + } + call.2 + .push_str(delta["function"]["arguments"].as_str().unwrap_or_default()); + } + } + calls +} + +fn finish_reason(chunks: &[Value]) -> Option { + chunks + .iter() + .find_map(|chunk| chunk["choices"][0]["finish_reason"].as_str()) + .map(str::to_owned) +} + +#[tokio::test] +async fn anthropic_tool_round_trip() { + let harness = harness().await; + + let asked = harness + .json(json!({ + "model": "claude", + "messages": [question()], + "tools": [weather_tool()], + "tool_choice": "required", + "max_tokens": 64, + })) + .await; + let message = &asked["choices"][0]["message"]; + assert_eq!(asked["choices"][0]["finish_reason"], "tool_calls"); + assert_eq!(message["content"], "Checking."); + assert_eq!( + message["tool_calls"], + json!([{"id": "toolu_1", "type": "function", "index": 1, + "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}}]) + ); + + let done = harness + .json(json!({ + "model": "claude", + "messages": answered("toolu_1"), + "tools": [weather_tool()], + "max_tokens": 64, + })) + .await; + assert_eq!(done["choices"][0]["message"]["content"], "Sunny."); + assert_eq!(done["choices"][0]["finish_reason"], "stop"); + + let sent = harness.sent(); + assert_eq!(sent[0].path, "/v1/messages"); + assert_eq!( + sent[0].body, + json!({ + "model": ANTHROPIC_MODEL, + "messages": [{"role": "user", "content": [{"type": "text", "text": "Weather in Paris?"}]}], + "max_tokens": 64, + "tools": [{"name": "get_weather", "type": "custom", "description": "The weather in a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}], + "tool_choice": {"type": "any"} + }) + ); + assert_eq!( + sent[1].body["messages"], + json!([ + {"role": "user", "content": [{"type": "text", "text": "Weather in Paris?"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny, 21C"}]} + ]) + ); +} + +#[tokio::test] +async fn bedrock_tool_round_trip() { + let harness = harness().await; + + let asked = harness + .json(json!({ + "model": "bedrock", + "messages": [question()], + "tools": [weather_tool()], + "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, + "max_tokens": 64, + })) + .await; + assert_eq!(asked["choices"][0]["finish_reason"], "tool_calls"); + assert_eq!( + asked["choices"][0]["message"]["tool_calls"], + json!([{"id": "tooluse_1", "type": "function", "index": 1, + "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}}]) + ); + + let done = harness + .json(json!({ + "model": "bedrock", + "messages": answered("tooluse_1"), + "tools": [weather_tool()], + "max_tokens": 64, + })) + .await; + assert_eq!(done["choices"][0]["message"]["content"], "Sunny."); + assert_eq!(done["choices"][0]["finish_reason"], "stop"); + + let sent = harness.sent(); + assert_eq!(sent[0].path, format!("/model/{BEDROCK_MODEL}/converse")); + assert_eq!( + sent[0].body["toolConfig"], + json!({ + "tools": [{"toolSpec": {"name": "get_weather", "description": "The weather in a city", + "inputSchema": {"json": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}}], + "toolChoice": {"tool": {"name": "get_weather"}} + }) + ); + assert_eq!( + sent[1].body["messages"], + json!([ + {"role": "user", "content": [{"text": "Weather in Paris?"}]}, + {"role": "assistant", "content": [{"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather", "input": {"city": "Paris"}}}]}, + {"role": "user", "content": [{"toolResult": {"toolUseId": "tooluse_1", "content": [{"text": "sunny, 21C"}]}}]} + ]) + ); +} + +#[rstest] +#[case::anthropic("claude", "toolu_1")] +#[case::bedrock("bedrock", "tooluse_1")] +#[tokio::test] +async fn a_streamed_tool_call_arrives_as_openai_tool_call_chunks( + #[case] model: &str, + #[case] id: &str, +) { + let harness = harness().await; + + let (status, body) = harness + .send(json!({ + "model": model, + "messages": [question()], + "tools": [weather_tool()], + "max_tokens": 64, + "stream": true, + })) + .await; + + assert_eq!(status, 200, "{body}"); + let (chunks, done) = frames(&body); + assert!(done, "{body}"); + assert_eq!( + streamed_tool_calls(&chunks), + [( + id.to_owned(), + "get_weather".to_owned(), + "{\"city\": \"Paris\"}".to_owned() + )] + ); + assert_eq!(finish_reason(&chunks).as_deref(), Some("tool_calls")); + let sent = &harness.sent()[0]; + let tools = match model { + "claude" => sent.body["tools"][0]["name"].clone(), + _ => sent.body["toolConfig"]["tools"][0]["toolSpec"]["name"].clone(), + }; + assert_eq!(tools, "get_weather"); +} + +#[rstest] +#[case::max_tokens("max_tokens")] +#[case::max_completion_tokens("max_completion_tokens")] +#[tokio::test] +async fn bedrock_takes_the_token_limit_in_its_inference_config(#[case] param: &str) { + let harness = harness().await; + + harness + .json(json!({ + "model": "bedrock", + "messages": [{"role": "system", "content": "Be brief."}, question()], + param: 32, + "temperature": 0.5, + "stop": ["END"], + })) + .await; + + let sent = &harness.sent()[0]; + assert_eq!( + sent.body["inferenceConfig"], + json!({"maxTokens": 32, "temperature": 0.5, "stopSequences": ["END"]}) + ); + assert_eq!(sent.body["system"], json!([{"text": "Be brief."}])); +} + +#[tokio::test] +async fn anthropic_gets_the_model_default_max_tokens_when_the_caller_names_none() { + let harness = harness().await; + + harness + .json(json!({"model": "claude", "messages": [question()]})) + .await; + + assert_eq!(harness.sent()[0].body["max_tokens"], 8192); +} + +#[tokio::test] +async fn an_image_data_url_reaches_each_provider_inline() { + let harness = harness().await; + let messages = json!([{"role": "user", "content": [ + {"type": "text", "text": "What is this?"}, + {"type": "image_url", "image_url": {"url": format!("data:image/png;base64,{PNG}")}} + ]}]); + + harness + .json(json!({"model": "claude", "messages": messages, "max_tokens": 16})) + .await; + harness + .json(json!({"model": "bedrock", "messages": messages, "max_tokens": 16})) + .await; + + let sent = harness.sent(); + assert_eq!( + sent[0].body["messages"][0]["content"][1], + json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG}}) + ); + assert_eq!( + sent[1].body["messages"][0]["content"][1], + json!({"image": {"format": "png", "source": {"bytes": PNG}}}) + ); +} + +#[tokio::test] +async fn anthropic_response_format_comes_back_as_the_answer() { + let harness = harness().await; + let schema = + json!({"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}); + + let answer = harness + .json(json!({ + "model": "claude", + "messages": [question()], + "max_tokens": 64, + "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": schema}}, + "user": "end-user-1", + })) + .await; + + assert_eq!( + answer["choices"][0]["message"]["content"], + "{\"city\": \"Paris\"}" + ); + assert_eq!(answer["choices"][0]["message"]["tool_calls"], Value::Null); + assert_eq!(answer["choices"][0]["finish_reason"], "stop"); + let sent = &harness.sent()[0]; + assert_eq!(sent.body["metadata"], json!({"user_id": "end-user-1"})); + assert_eq!(sent.body["tools"][0]["name"], "json_tool_call"); + assert_eq!(sent.body["tools"][0]["input_schema"], schema); + assert_eq!( + sent.body["tool_choice"], + json!({"name": "json_tool_call", "type": "tool"}) + ); +} + +#[rstest] +#[case::anthropic("claude", json!({"logprobs": true}))] +#[case::bedrock("bedrock", json!({"tool_choice": "none", "tools": [weather_tool()]}))] +#[case::n_choices("claude", json!({"n": 2}))] +#[tokio::test] +async fn an_unsupported_param_is_refused_and_dropped_with_drop_params( + #[case] model: &str, + #[case] extra: Value, +) { + let harness = harness().await; + let mut body = json!({"model": model, "messages": [question()], "max_tokens": 16}); + body.as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + + let (status, refused) = harness.send(body.clone()).await; + assert_eq!(status, 400, "{refused}"); + assert!( + harness.sent().is_empty(), + "a refused request never goes out" + ); + + body["drop_params"] = json!(true); + harness.json(body).await; + let sent = &harness.sent()[0]; + for name in extra.as_object().unwrap().keys() { + if name == "tools" { + continue; + } + assert!( + sent.body.get(name).is_none(), + "{name} reached {}", + sent.body + ); + } + assert!( + sent.body + .get("toolConfig") + .and_then(|config| config.get("toolChoice")) + .is_none() + ); + assert!(sent.body.get("drop_params").is_none()); +} + +#[tokio::test] +async fn a_deployment_that_drops_params_drops_them_without_the_request_asking() { + let harness = harness().await; + + harness + .json(json!({ + "model": "claude-dropping", + "messages": [question()], + "max_tokens": 16, + "logprobs": true, + })) + .await; + + assert!(harness.sent()[0].body.get("logprobs").is_none()); +} + +#[tokio::test] +async fn a_cache_control_part_reaches_anthropic_and_bedrock() { + let harness = harness().await; + let messages = json!([ + {"role": "system", "content": [{"type": "text", "text": "Long instructions.", "cache_control": {"type": "ephemeral"}}]}, + question() + ]); + + harness + .json(json!({"model": "claude", "messages": messages, "max_tokens": 16})) + .await; + harness + .json(json!({"model": "bedrock", "messages": messages, "max_tokens": 16})) + .await; + + let sent = harness.sent(); + assert_eq!( + sent[0].body["system"], + json!([{"type": "text", "text": "Long instructions.", "cache_control": {"type": "ephemeral"}}]) + ); + assert_eq!( + sent[1].body["system"], + json!([{"text": "Long instructions."}, {"cachePoint": {"type": "default"}}]) + ); + assert!(sent[0].header("x-api-key").is_some()); +} diff --git a/litellm-rust/crates/inference-chat/Cargo.toml b/litellm-rust/crates/inference-chat/Cargo.toml index 4cc32bb7656..aa2ca0f5e8a 100644 --- a/litellm-rust/crates/inference-chat/Cargo.toml +++ b/litellm-rust/crates/inference-chat/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true bytes.workspace = true futures-util.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index 5f4d00e1d7c..95764ccda42 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -29,7 +29,7 @@ pub(super) async fn execute( config, url, body, - stream_shape: _, + stream_shape, optional_params, environment, secrets, @@ -122,7 +122,7 @@ pub(super) async fn execute( )) })?; config - .transform_response(&model, ProviderChatResponseData { body }) + .transform_response(&model, ProviderChatResponseData { body }, &stream_shape) .map_err(Error::from) .map_err(as_response_error) }, @@ -229,6 +229,8 @@ mod tests { custom_llm_provider: None, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, }) .unwrap(), std::sync::Arc::new(|_: &str| None), diff --git a/litellm-rust/crates/inference-chat/src/lib.rs b/litellm-rust/crates/inference-chat/src/lib.rs index db4f28f4d2a..df442ea2489 100644 --- a/litellm-rust/crates/inference-chat/src/lib.rs +++ b/litellm-rust/crates/inference-chat/src/lib.rs @@ -8,6 +8,7 @@ pub use litellm_inference::RouteError as Error; mod common_utils; pub mod constants; pub(crate) mod handler; +mod media; pub mod openai; mod prepare; pub mod stream; @@ -25,6 +26,7 @@ pub struct ChatCompletionsRoute { auth: Arc, secrets: Arc, cache: Option, + media: Option, } impl ChatCompletionsRoute { @@ -38,6 +40,16 @@ impl ChatCompletionsRoute { auth, secrets, cache: None, + media: None, + } + } + + /// Downloads the media URLs a provider cannot take by reference. Without one such a + /// request is refused as unsupported. + pub fn with_media(self, media: litellm_http::media::MediaFetcher) -> Self { + Self { + media: Some(media), + ..self } } @@ -77,7 +89,8 @@ impl ChatCompletionsRoute { interceptors: &impl litellm_host::interceptors::Interceptors, observers: Option<&ObservationSender>, ) -> Result { - let resolved = resolve_request(request)?; + let mut resolved = resolve_request(request)?; + self.inline_remote_media(&mut resolved).await?; let snapshot = self .secrets .resolve(&resolved.config.secret_names()) diff --git a/litellm-rust/crates/inference-chat/src/media.rs b/litellm-rust/crates/inference-chat/src/media.rs new file mode 100644 index 00000000000..8287b496eba --- /dev/null +++ b/litellm-rust/crates/inference-chat/src/media.rs @@ -0,0 +1,158 @@ +//! The media a translated request needs downloaded first: an image or file URL the provider +//! cannot take by reference, which Python fetches while it builds the request. + +use std::{collections::HashMap, time::Duration}; + +use base64::Engine; +use futures_util::future::try_join_all; +use litellm_http::media::DownloadPolicy; +use litellm_llms::base_llm::chat::media::{ + MAX_MEDIA_DOWNLOAD_BYTES, downloaded_media_type, inline_media, +}; + +use crate::{ChatCompletionsRoute, Error, types::ResolvedChatCompletionsRequest}; + +const MEDIA_DOWNLOAD: DownloadPolicy = DownloadPolicy { + timeout: Duration::from_secs(60), + max_bytes: MAX_MEDIA_DOWNLOAD_BYTES, + max_redirects: 10, +}; + +impl ChatCompletionsRoute { + /// Downloads every URL the config names and inlines each as a data URL. A route with no + /// fetcher leaves such a request to the host; a URL that cannot be fetched is the + /// caller's error, as Python's image fetch failure is. + pub(crate) async fn inline_remote_media( + &self, + request: &mut ResolvedChatCompletionsRequest<'_>, + ) -> Result<(), Error> { + let urls = request.config.remote_media(&request.messages); + if urls.is_empty() { + return Ok(()); + } + let fetcher = self + .media + .as_ref() + .ok_or(Error::Unsupported("message media URL"))?; + let fetch = |url: String| async move { + let unable = |reason: String| { + Error::InvalidRequest( + format!("Unable to fetch media from url {url}: {reason}").into(), + ) + }; + let parsed = reqwest::Url::parse(&url).map_err(|err| unable(err.to_string()))?; + let media = fetcher + .fetch(parsed, MEDIA_DOWNLOAD) + .await + .map_err(|err| unable(err.to_string()))?; + let data = format!( + "data:{};base64,{}", + downloaded_media_type(&url, &media.content_type), + base64::engine::general_purpose::STANDARD.encode(media.bytes) + ); + Ok::<_, Error>((url, data)) + }; + let downloaded: HashMap = try_join_all(urls.into_iter().map(fetch)) + .await? + .into_iter() + .collect(); + inline_media(&mut request.messages, &downloaded); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use litellm_http::{Client, media::MediaFetcher}; + + use rstest::rstest; + use serde_json::{Value, json}; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }; + + use crate::{ + ChatCompletionsRoute, Error, prepare::resolve_request, types::ChatCompletionsRequest, + }; + + fn route(media: bool) -> ChatCompletionsRoute { + let route = crate::test_support::route(); + match media { + true => route.with_media(MediaFetcher::for_test(Client::plain_for_test())), + false => route, + } + } + + fn request(url: &str) -> ChatCompletionsRequest<'static> { + ChatCompletionsRequest { + model: "anthropic/claude-sonnet-4-5", + messages: json!([{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": url}} + ]}]), + optional_params: serde_json::Map::new(), + api_key: Some("sk-test"), + api_base: None, + custom_llm_provider: None, + extra_headers: None, + timeout: None, + drop_params: Some(false), + model_info: None, + } + } + + fn image(request: &crate::types::ResolvedChatCompletionsRequest<'_>) -> Value { + serde_json::to_value(&request.messages[0].content).unwrap()[0]["image_url"]["url"].clone() + } + + #[rstest] + #[tokio::test] + async fn a_remote_image_is_downloaded_and_inlined() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/cat.png")) + .respond_with(ResponseTemplate::new(200).set_body_raw(b"png".to_vec(), "image/png")) + .mount(&server) + .await; + let url = format!("{}/cat.png", server.uri()); + let url: &'static str = Box::leak(url.into_boxed_str()); + let mut resolved = resolve_request(request(url)).unwrap(); + + route(true) + .inline_remote_media(&mut resolved) + .await + .unwrap(); + + assert_eq!(image(&resolved), "data:image/png;base64,cG5n"); + } + + #[rstest] + #[tokio::test] + async fn a_media_url_that_cannot_be_fetched_is_the_callers_error() { + let server = MockServer::start().await; + let url = format!("{}/missing.png", server.uri()); + let url: &'static str = Box::leak(url.into_boxed_str()); + let mut resolved = resolve_request(request(url)).unwrap(); + + let error = route(true) + .inline_remote_media(&mut resolved) + .await + .unwrap_err(); + + assert!(matches!(error, Error::InvalidRequest(_)), "{error:?}"); + } + + #[rstest] + #[tokio::test] + async fn without_a_fetcher_a_media_url_is_unsupported_and_a_data_url_is_left_alone() { + let mut remote = resolve_request(request("http://example.com/cat.png")).unwrap(); + assert!(matches!( + route(false).inline_remote_media(&mut remote).await, + Err(Error::Unsupported(_)) + )); + + let mut inline = resolve_request(request("data:image/png;base64,cG5n")).unwrap(); + route(false).inline_remote_media(&mut inline).await.unwrap(); + assert_eq!(image(&inline), "data:image/png;base64,cG5n"); + } +} diff --git a/litellm-rust/crates/inference-chat/src/prepare.rs b/litellm-rust/crates/inference-chat/src/prepare.rs index 058ca245e79..5db550efd01 100644 --- a/litellm-rust/crates/inference-chat/src/prepare.rs +++ b/litellm-rust/crates/inference-chat/src/prepare.rs @@ -1,7 +1,10 @@ use litellm_auth::SecretValue; use litellm_core_utils::settings::Lookup; use litellm_http::request::with_default_headers; -use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig}; +use litellm_llms::base_llm::{ + auth::{Headers, ValidatedEnvironment}, + chat::{params::ChatRequestContext, transformation::BaseConfig}, +}; use litellm_llms_types::formats::chat_completions::ChatMessage; use litellm_secrets::source::Secrets; use serde_json::Value; @@ -72,9 +75,38 @@ pub(super) fn resolve_request( api_base: request.api_base, extra_headers: request.extra_headers, timeout: request.timeout, + drop_params: request.drop_params, + model_info: request.model_info, }) } +/// The headers a transformation asked for, added to the ones already going out. A header +/// already there (a forwarded `anthropic-beta`, say) keeps its values and gains the new ones, +/// comma separated, the way Python unions beta headers. +fn merge_headers(headers: &mut Headers, added: Headers) { + for (name, value) in added { + match headers + .iter_mut() + .find(|(existing, _)| existing.eq_ignore_ascii_case(&name)) + { + Some((_, existing)) => { + let mut values: Vec<&str> = existing + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .collect(); + for value in value.split(',').map(str::trim) { + if !value.is_empty() && !values.contains(&value) { + values.push(value); + } + } + *existing = values.join(", "); + } + None => headers.push((name, value)), + } + } +} + fn validate_environment( request: &ResolvedChatCompletionsRequest<'_>, model: &str, @@ -101,7 +133,7 @@ pub(super) fn prepare_provider_request( request: ResolvedChatCompletionsRequest<'_>, secrets: Secrets, ) -> Result { - let environment = + let mut environment = validate_environment(&request, &request.model, request.config, secrets.as_ref())?; let model = request.model; let config = request.config; @@ -112,8 +144,17 @@ pub(super) fn prepare_provider_request( &request.optional_params, &env_lookup, )?; - let transformed = - config.transform_request(&model, request.messages, request.optional_params.clone())?; + let context = ChatRequestContext { + drop_params: request.drop_params, + model_info: request.model_info.as_deref(), + }; + let transformed = config.transform_request( + &model, + request.messages, + request.optional_params.clone(), + &context, + )?; + merge_headers(&mut environment.headers, transformed.headers); Ok(ProviderChatCompletionsRequest { model, @@ -136,7 +177,7 @@ mod tests { use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth}; use serde_json::{Map, Value, json}; - use super::{prepare_provider_request, resolve_request}; + use super::{merge_headers, prepare_provider_request, resolve_request}; use crate::{ Error, types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}, @@ -183,6 +224,8 @@ mod tests { custom_llm_provider: provider, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, } } @@ -636,4 +679,27 @@ mod tests { "prepare did not carry the bearer token" ); } + + #[test] + fn transformation_headers_union_with_the_forwarded_ones() { + let mut headers = vec![ + ("Anthropic-Beta".to_owned(), "a,b".to_owned()), + ("x-other".to_owned(), "1".to_owned()), + ]; + merge_headers( + &mut headers, + vec![ + ("anthropic-beta".to_owned(), "b, c".to_owned()), + ("x-new".to_owned(), "2".to_owned()), + ], + ); + assert_eq!( + headers, + [ + ("Anthropic-Beta".to_owned(), "a, b, c".to_owned()), + ("x-other".to_owned(), "1".to_owned()), + ("x-new".to_owned(), "2".to_owned()), + ] + ); + } } diff --git a/litellm-rust/crates/inference-chat/src/route.rs b/litellm-rust/crates/inference-chat/src/route.rs index 50fc412a705..73eb69ddc0c 100644 --- a/litellm-rust/crates/inference-chat/src/route.rs +++ b/litellm-rust/crates/inference-chat/src/route.rs @@ -69,6 +69,8 @@ impl ChatCompletionsRoute { custom_llm_provider: call.custom_llm_provider.as_deref(), extra_headers: call.extra_headers, timeout: call.timeout, + drop_params: call.drop_params, + model_info: call.model_info, }; self.run(request, cache_options, interceptors, observers) .await diff --git a/litellm-rust/crates/inference-chat/src/stream.rs b/litellm-rust/crates/inference-chat/src/stream.rs index 33dd1d8a441..c7d15e9eb0a 100644 --- a/litellm-rust/crates/inference-chat/src/stream.rs +++ b/litellm-rust/crates/inference-chat/src/stream.rs @@ -140,7 +140,7 @@ impl ChatCompletionsRoute { let timeout = call .timeout .unwrap_or(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS)); - let resolved = resolve_request(ChatCompletionsRequest { + let mut resolved = resolve_request(ChatCompletionsRequest { model: &call.model, messages: call.messages, optional_params, @@ -149,7 +149,10 @@ impl ChatCompletionsRoute { custom_llm_provider: call.custom_llm_provider.as_deref(), extra_headers: call.extra_headers, timeout: call.timeout, + drop_params: call.drop_params, + model_info: call.model_info, })?; + self.inline_remote_media(&mut resolved).await?; let snapshot = self .secrets .resolve(&resolved.config.secret_names()) @@ -1019,6 +1022,8 @@ mod tests { custom_llm_provider: None, extra_headers: None, timeout: Some(Duration::from_secs(5)), + drop_params: Some(false), + model_info: None, }, response_model: "public/model".into(), usage: ReportedUsage::default(), diff --git a/litellm-rust/crates/inference-chat/src/types.rs b/litellm-rust/crates/inference-chat/src/types.rs index d0cb7b24d6c..e02b67e5d2b 100644 --- a/litellm-rust/crates/inference-chat/src/types.rs +++ b/litellm-rust/crates/inference-chat/src/types.rs @@ -1,20 +1,20 @@ use litellm_secrets::source::Secrets; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use litellm_auth::SecretValue; use litellm_llms::base_llm::{ auth::ValidatedEnvironment, chat::{streaming::StreamShape, transformation::BaseConfig}, + model_info::ModelInfo, }; use litellm_llms_types::formats::chat_completions::ChatMessage; use serde_json::{Map, Value}; /// A `/chat/completions` call as it crosses into the core. /// -/// `optional_params` arrives already mapped to the provider's own parameter -/// names by the host, exactly as the messages route receives an already -/// Anthropic-shaped body. The core owns the conversation translation, the -/// provider call, and the response normalization. +/// `optional_params` are the OpenAI request's own params; the provider config maps them, +/// refusing what the provider cannot take unless `drop_params` says to drop it. The core +/// owns the conversation translation, the provider call, and the response normalization. pub struct ChatCompletionsRequest<'a> { pub model: &'a str, pub messages: Value, @@ -24,6 +24,11 @@ pub struct ChatCompletionsRequest<'a> { pub custom_llm_provider: Option<&'a str>, pub extra_headers: Option>, pub timeout: Option, + /// `drop_params` from the deployment or the proxy settings, `None` when neither says. A + /// request's own `drop_params` rides in `optional_params`. + pub drop_params: Option, + /// The model cost map, for the param rules that read model capabilities. + pub model_info: Option>, } pub struct ChatCompletionsCall { @@ -35,6 +40,11 @@ pub struct ChatCompletionsCall { pub custom_llm_provider: Option, pub extra_headers: Option>, pub timeout: Option, + /// `drop_params` from the deployment or the proxy settings, `None` when neither says. A + /// request's own `drop_params` rides in `optional_params`. + pub drop_params: Option, + /// The model cost map, for the param rules that read model capabilities. + pub model_info: Option>, } impl From> for ChatCompletionsCall { @@ -48,6 +58,8 @@ impl From> for ChatCompletionsCall { custom_llm_provider: request.custom_llm_provider.map(str::to_owned), extra_headers: request.extra_headers, timeout: request.timeout, + drop_params: request.drop_params, + model_info: request.model_info, } } } @@ -62,6 +74,11 @@ pub struct ResolvedChatCompletionsRequest<'a> { pub api_base: Option<&'a str>, pub extra_headers: Option>, pub timeout: Option, + /// `drop_params` from the deployment or the proxy settings, `None` when neither says. A + /// request's own `drop_params` rides in `optional_params`. + pub drop_params: Option, + /// The model cost map, for the param rules that read model capabilities. + pub model_info: Option>, } pub struct ProviderChatCompletionsRequest { diff --git a/litellm-rust/crates/inference-chat/tests/caching.rs b/litellm-rust/crates/inference-chat/tests/caching.rs index 3fc432f8b46..69b6ed917be 100644 --- a/litellm-rust/crates/inference-chat/tests/caching.rs +++ b/litellm-rust/crates/inference-chat/tests/caching.rs @@ -84,6 +84,8 @@ async fn the_same_route_entrypoint_reports_facts_with_or_without_caching( custom_llm_provider: None, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, }, &(), Some(observer.clone()), @@ -254,6 +256,8 @@ async fn chat_cache_identity_follows_resolved_configuration_and_request_callback custom_llm_provider: None, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, }, &hooks, None, @@ -319,6 +323,8 @@ async fn signed_requests_bypass_response_caching(cache: Arc ChatCompletionsRequest<'static> { custom_llm_provider: None, extra_headers: None, timeout: Some(Duration::from_secs(10)), + drop_params: Some(false), + model_info: None, } } @@ -163,7 +165,7 @@ async fn bedrock_round_trip_is_signed_and_normalized(request: ChatCompletionsReq #[case::missing_usage( r#"{"model":"m","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn"}"# )] -#[case::tool_use_block(r#"{"model":"m","content":[{"type":"tool_use","id":"t","name":"f","input":{}}],"stop_reason":"tool_use","usage":{"input_tokens":1,"output_tokens":1}}"#)] +#[case::malformed_token_count(r#"{"model":"m","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","usage":{"input_tokens":"1","output_tokens":1}}"#)] #[case::not_json("not json")] #[tokio::test] async fn a_response_it_cannot_normalize_is_reported_as_already_sent( @@ -294,6 +296,8 @@ async fn direct_and_hosted_calls_share_hooks_and_lifecycle( custom_llm_provider: call.custom_llm_provider.as_deref(), extra_headers: call.extra_headers, timeout: call.timeout, + drop_params: Some(false), + model_info: None, }, &host, Some(host.events.0.sender.clone()), diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs index 44d66d5ebe6..5166d8b5ea6 100644 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs @@ -87,6 +87,7 @@ pub struct ChatCompletionsUsage { } #[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] pub struct ChatCompletionsChoiceMessage { pub role: String, // Whether an empty turn is `None` or `""` is the provider's choice, not a @@ -94,6 +95,14 @@ pub struct ChatCompletionsChoiceMessage { // while Converse assigns the joined string unconditionally. Each config // mirrors its own, so keep this optional and serialize it even when None. pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_blocks: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, } #[macro_rules_attribute::apply(crate::wire_type)] diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 2d3ea85d1fc..629b6ee75e1 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -22,6 +22,8 @@ litellm-secrets.workspace = true litellm-python-compat.workspace = true litellm-router-types.workspace = true base64.workspace = true +sha2.workspace = true +uuid.workspace = true bytes.workspace = true data-url = "0.3.2" futures-util.workspace = true diff --git a/litellm-rust/crates/llms/src/anthropic/chat/handler.rs b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs index a373e22873a..27eb184ed24 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/handler.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs @@ -4,8 +4,9 @@ //! Each event becomes one provider chunk, empty ones included; the host's stream wrapper //! (Python's `CustomStreamWrapper`) decides which of them the client sees. //! -//! DIVERGENCE: not built here, since the chat route never asks for what produces them: -//! `json_mode` / `response_format` tool conversion, `web_search_calls` and +//! DIVERGENCE: a `response_format` tool's answer is held until its block ends and sent as +//! one piece of content, where Python streams it as it decides the envelope; an answer +//! spread over several blocks is sent per block. Not built: `web_search_calls` and //! `code_interpreter_results` summaries, and a tool call's `caller`. The reasoning token //! estimate Python makes from streamed thinking text (with its tokenizer) is not made, so //! such a stream reports `completion_tokens_details` without counts unless Anthropic sends @@ -28,7 +29,14 @@ use serde_json::{Map, Value, json}; use crate::{ Error, - base_llm::{base_model_iterator::StreamTransformer, chat::streaming::StreamShape}, + base_llm::{ + base_model_iterator::StreamTransformer, + chat::{ + params::{RESPONSE_FORMAT_TOOL_NAME, python_dumps}, + response_format::{envelope_contents, json_tool_answer}, + streaming::StreamShape, + }, + }, }; /// Anthropic sends no status with an `error` event; Python raises it as a 500. @@ -51,6 +59,10 @@ pub struct ModelResponseIterator { web_search_results: Vec, tool_results: Vec, compaction_blocks: Vec, + /// Whether the tool block in progress is the `response_format` tool. + is_response_format_tool: bool, + converted_response_format_tool: bool, + response_format_buffer: String, } /// What one event contributes to its chunk. @@ -79,9 +91,58 @@ impl ModelResponseIterator { web_search_results: Vec::new(), tool_results: Vec::new(), compaction_blocks: Vec::new(), + is_response_format_tool: false, + converted_response_format_tool: false, + response_format_buffer: String::new(), } } + /// `_handle_json_mode_chunk`: the `response_format` tool's arguments become content. + fn handle_json_mode(&mut self, parsed: &mut Parsed) { + if !self.shape.json_mode { + return; + } + let Some(tool_use) = parsed.tool_use.take() else { + return; + }; + if tool_use.id.is_some() { + self.is_response_format_tool = + tool_use.function.name.as_deref() == Some(RESPONSE_FORMAT_TOOL_NAME); + if self.is_response_format_tool { + self.tool_index -= 1; + } + } + if !self.is_response_format_tool { + parsed.tool_use = Some(tool_use); + return; + } + self.converted_response_format_tool = true; + if self.shape.response_format_schema.is_some() { + match tool_use.id { + Some(_) => self.response_format_buffer.clear(), + None => self + .response_format_buffer + .push_str(&tool_use.function.arguments), + } + parsed.text = String::new(); + return; + } + parsed.text = json_tool_answer(&[tool_use.function.arguments], None); + } + + /// `_held_answer`: the held arguments, or what their envelope wraps. + fn flush_response_format_buffer(&mut self) -> String { + let held = std::mem::take(&mut self.response_format_buffer); + let Some(schema) = &self.shape.response_format_schema else { + return String::new(); + }; + serde_json::from_str::(&held) + .ok() + .and_then(|payload| envelope_contents(&payload, schema, RESPONSE_FORMAT_TOOL_NAME)) + .map(|contents| python_dumps(&contents)) + .unwrap_or(held) + } + /// Whether the tool block so far streamed only empty argument fragments. pub fn check_empty_tool_call_args(&self) -> bool { match self.content_blocks.first() { @@ -273,6 +334,10 @@ impl ModelResponseIterator { if in_tool && self.check_empty_tool_call_args() { parsed.tool_use = Some(self.tool_call(None, None, "{}".into())); } + if self.is_response_format_tool { + parsed.text = self.flush_response_format_buffer(); + } + self.is_response_format_tool = false; self.current_content_block_type = None; parsed } @@ -282,10 +347,16 @@ impl ModelResponseIterator { delta: MessagesDelta, usage: Option, ) -> Parsed { + let provider_finish_reason = + finish_reason_for(delta.stop_reason.as_deref().unwrap_or("stop")).to_string(); + let called_only_the_response_format_tool = provider_finish_reason == "tool_calls" + && self.converted_response_format_tool + && self.tool_index < 0; let mut parsed = Parsed { - finish_reason: Some( - finish_reason_for(delta.stop_reason.as_deref().unwrap_or("stop")).to_string(), - ), + finish_reason: Some(match called_only_the_response_format_tool { + true => "stop".to_owned(), + false => provider_finish_reason, + }), usage: usage.map(|usage| self.handle_usage(&usage)), ..Parsed::default() }; @@ -314,7 +385,7 @@ impl ModelResponseIterator { &mut self, event: MessagesStreamEvent, ) -> Result { - let parsed = match event { + let mut parsed = match event { MessagesStreamEvent::ContentBlockDelta { delta, .. } => self.content_block_delta(delta), MessagesStreamEvent::ContentBlockStart { content_block, .. } => { self.content_block_start(content_block) @@ -341,6 +412,7 @@ impl ModelResponseIterator { }); } }; + self.handle_json_mode(&mut parsed); Ok(self.chunk(parsed)) } diff --git a/litellm-rust/crates/llms/src/anthropic/chat/messages.rs b/litellm-rust/crates/llms/src/anthropic/chat/messages.rs new file mode 100644 index 00000000000..8f4559f1e73 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/chat/messages.rs @@ -0,0 +1,777 @@ +//! Chat messages as the Messages API takes them: Python's `translate_system_message`, +//! `place_mid_conversation_system` and `anthropic_messages_pt`. + +use std::collections::HashSet; + +use base64::Engine; +use serde_json::{Map, Value, json}; + +use crate::{ + Error, ErrorDetail, + anthropic::common_utils::ENCRYPTED_REASONING_SIGNATURE_PREFIX, + base_llm::chat::{ + media::{data_url, image_url}, + params::{parse_tool_arguments, truthy}, + }, +}; + +pub const EMPTY_TEXT_PLACEHOLDER: &str = + "[System: Empty message content sanitised to satisfy protocol]"; +pub const CONVERTED_SYSTEM_NOTE: &str = "Operator note (not from the user): the following was originally a mid-conversation system-role reminder."; +const TOOL_USE_ID_FALLBACK: &str = "tool_use_id"; + +fn bad_request(message: impl Into) -> Error { + Error::InvalidRequest(ErrorDetail::Message(message.into())) +} + +pub fn role(message: &Value) -> &str { + match message.get("role").and_then(Value::as_str) { + Some("developer") => "system", + Some(role) => role, + None => "", + } +} + +/// Python's truthiness of a message's `content`. +fn content_is_empty(message: &Value) -> bool { + match message.get("content") { + None | Some(Value::Null) => true, + Some(Value::String(text)) => text.is_empty(), + Some(Value::Array(items)) => items.is_empty(), + Some(_) => false, + } +} + +/// `[^a-zA-Z0-9_-]` -> `_`, an empty id becoming `tool_use_id`. +pub fn sanitize_tool_use_id(id: &str) -> String { + let sanitized: String = id + .chars() + .map( + |c| match c.is_ascii_alphanumeric() || c == '_' || c == '-' { + true => c, + false => '_', + }, + ) + .collect(); + match sanitized.is_empty() { + true => TOOL_USE_ID_FALLBACK.to_owned(), + false => sanitized, + } +} + +/// `add_cache_control_to_content`: a dict `cache_control` on the source is copied over. +fn with_cache_control(mut block: Value, source: &Value) -> Value { + if let (Some(cache_control @ Value::Object(_)), Value::Object(fields)) = + (source.get("cache_control"), &mut block) + { + fields.insert("cache_control".into(), cache_control.clone()); + } + block +} + +/// `_cache_control` of the mid-conversation module: only an ephemeral one, rebuilt. +fn normalized_cache_control(source: &Value) -> Option { + let cache_control = source.get("cache_control")?.as_object()?; + if cache_control.get("type").and_then(Value::as_str) != Some("ephemeral") { + return None; + } + Some(match cache_control.get("ttl").and_then(Value::as_str) { + Some(ttl @ ("1h" | "5m")) => json!({"type": "ephemeral", "ttl": ttl}), + _ => json!({"type": "ephemeral"}), + }) +} + +/// The leading run of system messages, and the rest. +pub fn split_leading_system_run(messages: Vec) -> (Vec, Vec) { + let leading = messages + .iter() + .position(|message| role(message) != "system") + .unwrap_or(messages.len()); + let mut messages = messages; + let rest = messages.split_off(leading); + (messages, rest) +} + +/// `translate_system_message` for the leading run. +pub fn system_blocks(run: &[Value]) -> Vec { + let mut blocks = Vec::new(); + for message in run { + match message.get("content") { + Some(Value::String(text)) => { + if text.is_empty() { + continue; + } + let mut block = Map::from_iter([ + ("type".to_owned(), json!("text")), + ("text".to_owned(), json!(text)), + ]); + if let Some(cache_control) = message.get("cache_control") { + block.insert("cache_control".into(), cache_control.clone()); + } + blocks.push(Value::Object(block)); + } + Some(Value::Array(parts)) => { + for part in parts { + let text = part.get("text").cloned().unwrap_or(Value::Null); + let kind = part.get("type").cloned().unwrap_or(Value::Null); + let empty = match &text { + Value::String(text) => text.is_empty(), + Value::Null => true, + _ => false, + }; + if kind.as_str() == Some("text") && empty { + continue; + } + let mut block = + Map::from_iter([("type".to_owned(), kind), ("text".to_owned(), text)]); + if let Some(cache_control) = part.get("cache_control") { + block.insert("cache_control".into(), cache_control.clone()); + } + blocks.push(Value::Object(block)); + } + } + _ => {} + } + } + blocks +} + +/// The non-empty text parts of a system message, each with its normalized `cache_control`. +fn system_text_parts(message: &Value) -> Vec { + let block = |text: &str, cache_control: Option| { + let mut block = Map::from_iter([ + ("type".to_owned(), json!("text")), + ("text".to_owned(), json!(text)), + ]); + if let Some(cache_control) = cache_control { + block.insert("cache_control".into(), cache_control); + } + Value::Object(block) + }; + match message.get("content") { + Some(Value::String(text)) if !text.is_empty() => { + vec![block(text, normalized_cache_control(message))] + } + Some(Value::Array(parts)) => parts + .iter() + .filter(|part| part.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|part| { + let text = part.get("text")?.as_str().filter(|text| !text.is_empty())?; + Some(block(text, normalized_cache_control(part))) + }) + .collect(), + _ => Vec::new(), + } +} + +/// `system_message_as_user` for each message of a run that carries text. +pub fn converted_user_turns(run: &[Value]) -> Vec { + run.iter() + .filter_map(|message| { + let parts = system_text_parts(message); + if parts.is_empty() { + return None; + } + let mut content = vec![json!({"type": "text", "text": CONVERTED_SYSTEM_NOTE})]; + content.extend(parts); + Some(json!({"role": "user", "content": content})) + }) + .collect() +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + System, + Tool, + User, + Other, +} + +fn kind(message: &Value) -> Kind { + match role(message) { + "system" => Kind::System, + "tool" | "function" => Kind::Tool, + "user" => Kind::User, + _ => Kind::Other, + } +} + +/// `_converted_for_unflagged_model`: every later system message becomes a user turn in +/// place, a run caught before a tool result moving behind it. +pub fn place_mid_conversation_system(messages: Vec) -> Vec { + if !messages.iter().any(|message| kind(message) == Kind::System) { + return messages; + } + let mut runs: Vec<(Kind, Vec)> = Vec::new(); + for message in messages { + let message_kind = kind(&message); + match runs.last_mut() { + Some((last, run)) if *last == message_kind => run.push(message), + _ => runs.push((message_kind, vec![message])), + } + } + let mut placed = Vec::new(); + for index in 0..runs.len() { + let (run_kind, run) = &runs[index]; + let follower = runs.get(index + 1).map(|(kind, _)| *kind); + match run_kind { + Kind::System => { + if follower != Some(Kind::Tool) { + placed.extend(converted_user_turns(run)); + } + } + Kind::Tool if index > 0 && runs[index - 1].0 == Kind::System => { + placed.extend(run.iter().cloned()); + placed.extend(converted_user_turns(&runs[index - 1].1)); + } + _ => placed.extend(run.iter().cloned()), + } + } + placed +} + +/// `_sanitize_empty_text_content`. +fn sanitize_empty_text(mut message: Value) -> Value { + if !matches!(role(&message), "user" | "assistant") { + return message; + } + match message.get_mut("content") { + Some(Value::String(text)) if text.trim().is_empty() => { + *text = EMPTY_TEXT_PLACEHOLDER.to_owned(); + } + Some(Value::Array(parts)) => { + for part in parts { + if part.get("type").and_then(Value::as_str) != Some("text") { + continue; + } + let blank = part + .get("text") + .and_then(Value::as_str) + .is_none_or(|text| text.trim().is_empty()); + if blank && let Value::Object(fields) = part { + fields.insert("text".into(), json!(EMPTY_TEXT_PLACEHOLDER)); + } + } + } + _ => {} + } + message +} + +/// `convert_to_anthropic_image_obj`: a data URL as its media type and payload. +fn image_source(url: &str, format: Option<&str>) -> Result<(String, String), Error> { + let (media_type, data) = data_url(url).ok_or_else(|| { + bad_request(format!( + "Image url not in expected format. Example Expected input - \"image_url\": \"data:image/jpeg;base64,{{base64_image}}\". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp']. Error: {url:.40}" + )) + })?; + let media_type = match format { + Some(format) => format.to_owned(), + None => media_type.replace("\\/", "/"), + }; + Ok((media_type, data.to_owned())) +} + +/// `create_anthropic_image_param`. +fn image_block(part: &Value) -> Result { + let url = image_url(part).unwrap_or_default(); + let format = part + .get("image_url") + .and_then(|image| image.get("format")) + .and_then(Value::as_str); + if url.starts_with("https://") { + return Ok(json!({"type": "image", "source": {"type": "url", "url": url}})); + } + if url.starts_with("http") { + return Err(Error::Unsupported( + "an http:// image the host did not download", + )); + } + let (media_type, data) = image_source(url, format)?; + Ok( + json!({"type": "image", "source": {"type": "base64", "media_type": media_type, "data": data}}), + ) +} + +/// `anthropic_process_openai_file_message`. +fn file_block(part: &Value) -> Result { + let file = part.get("file").ok_or_else(|| { + bad_request("Content block has type='file' but is missing the required 'file' field") + })?; + let format = file.get("format").and_then(Value::as_str); + if let Some(data) = file + .get("file_data") + .and_then(Value::as_str) + .filter(|data| !data.is_empty()) + { + if data.starts_with("http") { + return Err(Error::Unsupported( + "a file_data URL the host did not download", + )); + } + let (media_type, data) = image_source(data, format)?; + return Ok( + json!({"type": "document", "source": {"type": "base64", "media_type": media_type, "data": data}}), + ); + } + match file + .get("file_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + { + Some(id) if format.is_none() && id.starts_with("http") => { + Ok(json!({"type": "document", "source": {"type": "url", "url": id}})) + } + // Uploaded files need the Files API betas and, for a container upload, the code + // execution tool. + Some(_) => Err(Error::Unsupported("an Anthropic Files API file_id")), + None => Err(bad_request(format!( + "Either file_data or file_id must be present in the file message: {part}" + ))), + } +} + +/// `_is_anthropic_document_data_uri`. +fn is_document_data_uri(url: &str) -> bool { + url.strip_prefix("data:") + .map(|rest| rest.split([';', ',']).next().unwrap_or_default()) + .is_some_and(|media_type| matches!(media_type, "application/pdf" | "text/plain")) +} + +/// `convert_to_anthropic_tool_result`. +fn tool_result(message: &Value) -> Result { + let content = match message.get("content") { + Some(Value::String(text)) => json!(text), + Some(Value::Array(parts)) => { + let mut blocks = Vec::new(); + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("text") => { + let mut block = Map::from_iter([ + ("type".to_owned(), json!("text")), + ( + "text".to_owned(), + part.get("text").cloned().unwrap_or(Value::Null), + ), + ]); + if let Some(cache_control) = + part.get("cache_control").filter(|value| !value.is_null()) + { + block.insert("cache_control".into(), cache_control.clone()); + } + blocks.push(Value::Object(block)); + } + Some("image_url") => { + let url = image_url(part).unwrap_or_default(); + let block = match is_document_data_uri(url) { + true => { + file_block(&json!({"type": "file", "file": {"file_data": url}}))? + } + false => image_block(part)?, + }; + blocks.push(with_cache_control(block, part)); + } + Some("tool_reference") => blocks.push(json!({ + "type": "tool_reference", + "tool_name": part.get("tool_name").cloned().unwrap_or(Value::Null), + })), + Some("file") => blocks.push(with_cache_control(file_block(part)?, part)), + _ => {} + } + } + Value::Array(blocks) + } + _ => json!(""), + }; + let id = match role(message) { + "tool" => message + .get("tool_call_id") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| bad_request("A tool message needs a tool_call_id."))?, + _ => message + .get("tool_call_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()), + }; + let mut result = Map::from_iter([ + ("type".to_owned(), json!("tool_result")), + ("tool_use_id".to_owned(), json!(sanitize_tool_use_id(&id))), + ("content".to_owned(), content), + ]); + if let Some(cache_control) = message + .get("cache_control") + .filter(|value| !value.is_null()) + { + result.insert("cache_control".into(), cache_control.clone()); + } + Ok(Value::Object(result)) +} + +pub fn is_encrypted_reasoning(block: &Value) -> bool { + let field = match block.get("type").and_then(Value::as_str) { + Some("thinking") => block.get("signature"), + Some("redacted_thinking") => block.get("data"), + _ => None, + }; + field + .and_then(Value::as_str) + .is_some_and(|value| value.starts_with(ENCRYPTED_REASONING_SIGNATURE_PREFIX)) +} + +/// `is_unsignable_thinking_block`. +pub fn is_unsignable_thinking_block(block: &Value) -> bool { + if is_encrypted_reasoning(block) { + return true; + } + block.get("type").and_then(Value::as_str) == Some("thinking") + && block + .get("signature") + .and_then(Value::as_str) + .is_none_or(str::is_empty) +} + +/// `native_redacted_thinking_data`: the base64 wrapper Converse adds, removed. +pub fn native_redacted_thinking_data(data: &str) -> String { + let engine = base64::engine::general_purpose::STANDARD; + engine + .decode(data) + .ok() + .and_then(|bytes| String::from_utf8(bytes).ok()) + .filter(|payload| payload.is_ascii() && engine.decode(payload).is_ok()) + .unwrap_or_else(|| data.to_owned()) +} + +/// `_replayable_thinking_blocks`. +fn replayable_thinking_blocks(blocks: &[Value]) -> Vec { + blocks + .iter() + .filter(|block| !is_unsignable_thinking_block(block)) + .map(|block| { + let mut block = block.clone(); + if let Value::Object(fields) = &mut block { + if fields.get("type").and_then(Value::as_str) == Some("redacted_thinking") + && let Some(Value::String(data)) = fields.get("data") + { + let native = native_redacted_thinking_data(data); + fields.insert("data".into(), json!(native)); + } + fields.remove("index"); + } + block + }) + .collect() +} + +/// The tool calls of an assistant message, as `tool_use` blocks. +fn tool_uses(tool_calls: &[Value], seen: &mut HashSet) -> Result, Error> { + let mut blocks = Vec::new(); + for call in tool_calls { + if call.get("type").and_then(Value::as_str) != Some("function") { + continue; + } + let id = call.get("id").and_then(Value::as_str).unwrap_or_default(); + if id.starts_with("srvtoolu_") { + return Err(Error::Unsupported( + "replaying an Anthropic server tool call", + )); + } + let function = call.get("function"); + let name = function + .and_then(|function| function.get("name")) + .cloned() + .unwrap_or(Value::Null); + let input = parse_tool_arguments(function.and_then(|function| function.get("arguments")))?; + let id = sanitize_tool_use_id(id); + let block = with_cache_control( + json!({"type": "tool_use", "id": id, "name": name, "input": input}), + call, + ); + if !seen.insert(id) { + continue; + } + blocks.push(block); + } + Ok(blocks) +} + +/// `anthropic_messages_pt` over messages whose later system messages are already placed. +pub fn anthropic_messages_pt(messages: Vec) -> Result, Error> { + let messages: Vec = messages.into_iter().map(sanitize_empty_text).collect(); + if messages.is_empty() { + return Err(bad_request( + "Anthropic requires at least one non-system message. Either provide one, or set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add the dummy user message - {'role': 'user', 'content': 'Please continue.'}.", + )); + } + let user_types = |message: &Value| matches!(role(message), "user" | "tool" | "function"); + // `_tool_result_later_in_turn`: whether a client tool result follows in the same turn. + let mut tool_result_later = vec![false; messages.len()]; + let mut later = false; + for index in (0..messages.len()).rev() { + tool_result_later[index] = later; + let message = &messages[index]; + later = user_types(message) && (later || matches!(role(message), "tool" | "function")); + } + + let mut out: Vec = Vec::new(); + let mut index = 0; + while index < messages.len() { + let start = index; + let mut user_content: Vec = Vec::new(); + while index < messages.len() && user_types(&messages[index]) { + let message = &messages[index]; + if role(message) == "user" { + match message.get("content") { + _ if content_is_empty(message) => { + if !tool_result_later[index] + && message.get("cache_control").is_some_and(truthy) + { + user_content.push(with_cache_control( + json!({"type": "text", "text": EMPTY_TEXT_PLACEHOLDER}), + message, + )); + } + } + Some(Value::Array(parts)) => { + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("image_url") => { + user_content.push(with_cache_control(image_block(part)?, part)) + } + Some("text") => user_content.push(with_cache_control( + json!({"type": "text", "text": part.get("text").cloned().unwrap_or(Value::Null)}), + part, + )), + Some("document") => { + user_content.push(with_cache_control(part.clone(), part)) + } + Some("file") => { + user_content.push(with_cache_control(file_block(part)?, part)) + } + _ => {} + } + } + } + Some(Value::String(text)) => user_content.push(with_cache_control( + json!({"type": "text", "text": text}), + message, + )), + _ => {} + } + } else { + user_content.push(tool_result(message)?); + } + index += 1; + } + if !user_content.is_empty() { + out.push(json!({"role": "user", "content": user_content})); + } else if let Some(empty) = messages[start..index] + .iter() + .rev() + .find(|message| role(message) == "user" && content_is_empty(message)) + { + out.push(json!({"role": "user", "content": [with_cache_control( + json!({"type": "text", "text": EMPTY_TEXT_PLACEHOLDER}), + empty, + )]})); + } + + let mut seen_tool_ids: HashSet = HashSet::new(); + let mut assistant_content: Vec = Vec::new(); + while index < messages.len() && role(&messages[index]) == "assistant" { + let message = &messages[index]; + let provider_fields = message.get("provider_specific_fields"); + if let Some(Value::Array(blocks)) = + provider_fields.and_then(|fields| fields.get("compaction_blocks")) + { + assistant_content.extend(blocks.iter().cloned()); + } + if provider_fields.is_some_and(|fields| { + fields + .get("web_search_results") + .is_some_and(|v| !v.is_null()) + || fields.get("tool_results").is_some_and(|v| !v.is_null()) + }) && message + .get("tool_calls") + .and_then(Value::as_array) + .is_some_and(|calls| !calls.is_empty()) + { + return Err(Error::Unsupported( + "replaying Anthropic server tool results", + )); + } + let thinking_blocks = match message.get("thinking_blocks") { + Some(Value::Array(blocks)) => Some(replayable_thinking_blocks(blocks)), + _ => None, + }; + let parts = message.get("content").and_then(Value::as_array); + let inline_thinking = parts.is_some_and(|parts| { + parts.iter().any(|part| { + matches!( + part.get("type").and_then(Value::as_str), + Some("thinking" | "redacted_thinking") + ) + }) + }); + if let Some(blocks) = thinking_blocks + && !inline_thinking + { + assistant_content.extend(blocks); + } + match message.get("content") { + Some(Value::Array(parts)) => { + for part in parts.iter().filter(|part| part.is_object()) { + let kind = part.get("type").and_then(Value::as_str).unwrap_or_default(); + let text = part.get("text").and_then(Value::as_str).unwrap_or_default(); + if kind == "thinking" && !is_unsignable_thinking_block(part) { + assistant_content.push(part.clone()); + } else if kind == "text" && !text.is_empty() { + assistant_content.push(with_cache_control( + json!({"type": "text", "text": text}), + part, + )); + } else if kind == "server_tool_use" || kind.ends_with("_tool_result") { + assistant_content.push(part.clone()); + } + } + } + Some(Value::String(text)) if !text.is_empty() => { + assistant_content.push(with_cache_control( + json!({"type": "text", "text": text}), + message, + )); + } + _ => {} + } + if let Some(Value::Array(calls)) = message.get("tool_calls") { + assistant_content.extend(tool_uses(calls, &mut seen_tool_ids)?); + } + if let Some(call) = message.get("function_call").filter(|call| !call.is_null()) { + let input = parse_tool_arguments(call.get("arguments"))?; + assistant_content.push(json!({ + "type": "tool_use", + "id": uuid::Uuid::new_v4().to_string(), + "name": call.get("name").filter(|name| !name.is_null()).cloned().unwrap_or_else(|| json!("")), + "input": input, + })); + } + index += 1; + } + if !assistant_content.is_empty() { + out.push(json!({"role": "assistant", "content": assistant_content})); + } + if index == start { + return Err(bad_request(format!( + "Invalid Message passed in {}", + messages[index] + ))); + } + } + if let Some(last) = out.last_mut() + && last["role"] == "assistant" + && let Some(Value::Array(blocks)) = last.get_mut("content") + { + for block in blocks { + if block.get("type").and_then(Value::as_str) == Some("text") + && let Some(Value::String(text)) = block.get_mut("text") + { + *text = text.trim_end().to_owned(); + } + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn list(value: Value) -> Vec { + value.as_array().unwrap().clone() + } + + #[rstest] + fn tool_round_trip_merges_turns() { + let messages = anthropic_messages_pt(list(json!([ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": null, "tool_calls": [{"id": "call.1", "type": "function", "function": {"name": "get", "arguments": "{\"c\": \"P\"}"}}]}, + {"role": "tool", "tool_call_id": "call.1", "content": "sunny"}, + {"role": "user", "content": [{"type": "text", "text": "thanks", "cache_control": {"type": "ephemeral"}}]} + ]))) + .unwrap(); + assert_eq!( + messages, + list(json!([ + {"role": "user", "content": [{"type": "text", "text": "weather?"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "get", "input": {"c": "P"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}, + {"type": "text", "text": "thanks", "cache_control": {"type": "ephemeral"}} + ]} + ])) + ); + } + + #[rstest] + fn images_empty_text_and_final_assistant_whitespace() { + let messages = anthropic_messages_pt(list(json!([ + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBO"}}, + {"type": "image_url", "image_url": "https://x/y.png"}, + {"type": "text", "text": " "} + ]}, + {"role": "assistant", "content": "Sure "} + ]))) + .unwrap(); + assert_eq!( + messages[0]["content"], + json!([ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBO"}}, + {"type": "image", "source": {"type": "url", "url": "https://x/y.png"}}, + {"type": "text", "text": EMPTY_TEXT_PLACEHOLDER} + ]) + ); + assert_eq!( + messages[1]["content"], + json!([{"type": "text", "text": "Sure"}]) + ); + } + + #[rstest] + fn later_system_messages_become_operator_notes() { + let placed = place_mid_conversation_system(list(json!([ + {"role": "user", "content": "a"}, + {"role": "assistant", "content": "b", "tool_calls": [{"id": "t", "type": "function", "function": {"name": "f", "arguments": "{}"}}]}, + {"role": "system", "content": "remember", "cache_control": {"type": "ephemeral", "ttl": "1h", "x": 1}}, + {"role": "tool", "tool_call_id": "t", "content": "r"} + ]))); + assert_eq!(placed[2]["role"], "tool"); + assert_eq!( + placed[3], + json!({"role": "user", "content": [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "text", "text": "remember", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ]}) + ); + } + + #[rstest] + fn unsigned_thinking_is_dropped_and_signed_thinking_leads() { + let messages = anthropic_messages_pt(list(json!([ + {"role": "user", "content": "q"}, + {"role": "assistant", "content": "a", "thinking_blocks": [ + {"type": "thinking", "thinking": "t", "signature": "s", "index": 0}, + {"type": "thinking", "thinking": "u"} + ]} + ]))) + .unwrap(); + assert_eq!( + messages[1]["content"], + json!([{"type": "thinking", "thinking": "t", "signature": "s"}, {"type": "text", "text": "a"}]) + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/chat/mod.rs b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs index f0050b7dc71..955055f2fbd 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs @@ -1,2 +1,4 @@ pub mod handler; +pub mod messages; +pub mod params; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/anthropic/chat/params.rs b/litellm-rust/crates/llms/src/anthropic/chat/params.rs new file mode 100644 index 00000000000..be507b1c46c --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/chat/params.rs @@ -0,0 +1,1293 @@ +//! Python's `AnthropicConfig.map_openai_params` and the model-map reads behind it: the +//! OpenAI params a chat request carries, as the Messages API takes them. + +use serde_json::{Map, Value, json}; + +use crate::{ + Error, ErrorDetail, + base_llm::chat::{ + params::{ + ChatRequestContext, DEFAULT_MAX_TOKENS, DropPolicy, RESPONSE_FORMAT_TOOL_NAME, + capability, ordered, truthy, unsupported_value, + }, + response_format::{ResponseFormatSchema, unpack_defs, unpack_legacy_defs}, + }, + bedrock::chat::converse_params::bedrock_base_model, +}; + +/// `ANTHROPIC_MIN_THINKING_BUDGET_TOKENS`. +pub const MIN_THINKING_BUDGET_TOKENS: i64 = 1024; + +/// The thinking budgets `_map_reasoning_effort` gives a legacy-thinking model. +pub fn reasoning_effort_budget(effort: &str) -> Option { + Some(match effort { + "low" => 1024, + "medium" => 2048, + "high" => 4096, + "xhigh" => 8192, + "max" => 16384, + // `max(DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS)` + "minimal" => 128.max(MIN_THINKING_BUDGET_TOKENS), + _ => return None, + }) +} + +/// `REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT`. +pub fn output_config_effort(effort: &str) -> Option<&'static str> { + Some(match effort { + "low" | "minimal" => "low", + "medium" => "medium", + "high" => "high", + "xhigh" => "xhigh", + "max" => "max", + _ => return None, + }) +} + +pub fn unmapped_reasoning_effort(effort: &str) -> Error { + Error::InvalidRequest(ErrorDetail::InvalidChoice { + field: "reasoning effort", + actual: format!("'{effort}'"), + choices: vec!["minimal", "low", "medium", "high", "xhigh", "max", "none"], + }) +} + +/// Strips a trailing `-v` or `-v:`. +fn strip_bedrock_version(model: &str) -> String { + let bytes = model.as_bytes(); + let mut end = bytes.len(); + // Optional `:`. + let mut cut = end; + while cut > 0 && bytes[cut - 1].is_ascii_digit() { + cut -= 1; + } + if cut < end && cut > 0 && bytes[cut - 1] == b':' { + end = cut - 1; + } + let mut digits = end; + while digits > 0 && bytes[digits - 1].is_ascii_digit() { + digits -= 1; + } + if digits < end && digits >= 2 && &bytes[digits - 2..digits] == b"-v" { + return model[..digits - 2].to_owned(); + } + // The `:` alone is not a version suffix, so only the full shape is removed. + model.to_owned() +} + +/// Strips a trailing `:`. +fn strip_profile_minor(model: &str) -> String { + let trimmed = model.trim_end_matches(|c: char| c.is_ascii_digit()); + match trimmed.strip_suffix(':') { + Some(head) if trimmed.len() < model.len() => head.to_owned(), + _ => model.to_owned(), + } +} + +/// Strips a trailing `-<8 digits>`. +fn strip_dated_release(model: &str) -> String { + let bytes = model.as_bytes(); + if bytes.len() >= 9 { + let tail = &bytes[bytes.len() - 9..]; + if tail[0] == b'-' && tail[1..].iter().all(u8::is_ascii_digit) { + return model[..model.len() - 9].to_owned(); + } + } + model.to_owned() +} + +/// `(\d)\.(\d)` -> `\1-\2`, non-overlapping like `re.sub`. +fn hyphenate_dotted_versions(model: &str) -> String { + let chars: Vec = model.chars().collect(); + let mut out = String::with_capacity(model.len()); + let mut index = 0; + while index < chars.len() { + if index + 2 < chars.len() + && chars[index].is_ascii_digit() + && chars[index + 1] == '.' + && chars[index + 2].is_ascii_digit() + { + out.push(chars[index]); + out.push('-'); + out.push(chars[index + 2]); + index += 3; + } else { + out.push(chars[index]); + index += 1; + } + } + out +} + +fn strip_at_version(model: &str) -> String { + match model.rfind('@') { + Some(at) if at > 0 => model[..at].to_owned(), + _ => model.to_owned(), + } +} + +/// `AnthropicModelInfo._model_map_lookup_candidates`. +pub fn model_map_candidates(model: &str) -> Vec { + let mut primary = vec![model.to_owned()]; + for prefix in [ + "bedrock/converse/", + "bedrock/invoke/", + "bedrock/", + "vertex_ai/", + ] { + if let Some(rest) = model.strip_prefix(prefix) { + primary.push(rest.to_owned()); + } + } + let base = bedrock_base_model(model); + if !base.is_empty() { + primary.push(base.clone()); + primary.push(format!("bedrock/{base}")); + } + let mut candidates: Vec = Vec::new(); + let mut push = |candidate: String| { + if !candidates.contains(&candidate) { + candidates.push(candidate); + } + }; + for candidate in &primary { + push(candidate.clone()); + } + for candidate in &primary { + push(strip_bedrock_version(candidate)); + push(strip_profile_minor(candidate)); + push(strip_dated_release(candidate)); + push(hyphenate_dotted_versions(candidate)); + push(strip_dated_release(&strip_bedrock_version( + &hyphenate_dotted_versions(candidate), + ))); + push(strip_at_version(candidate)); + } + candidates +} + +/// The cost-map reads a model's request rules make, through its lookup candidates. +pub struct ModelCapabilities<'a> { + pub model: String, + pub context: ChatRequestContext<'a>, + pub candidates: Vec, +} + +impl<'a> ModelCapabilities<'a> { + pub fn new(model: &str, context: &ChatRequestContext<'a>) -> Self { + Self { + model: model.to_owned(), + context: *context, + candidates: model_map_candidates(model), + } + } + + /// `_get_model_capability`: the flag the first candidate the map knows sets. + pub fn flag(&self, key: &str) -> Result, Error> { + capability(&self.context, &self.candidates, key) + } + + /// `_supports_model_capability`: the flag is set to `true`. + pub fn supports(&self, key: &str) -> Result { + Ok(self.flag(key)? == Some(true)) + } + + /// `_supports_sampling_params`. + pub fn supports_sampling(&self) -> Result { + if let Some(flag) = self.flag("supports_sampling_params")? { + return Ok(flag); + } + let lower = self.model.to_lowercase(); + Ok(![ + "fable", "opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7", "opus-4-8", "opus_4_8", + "opus-4.8", "opus_4.8", + ] + .iter() + .any(|name| lower.contains(name))) + } + + pub fn forced_tool_use_unsupported(&self) -> Result { + Ok(self.flag("supports_forced_tool_use")? == Some(false)) + } + + pub fn is_adaptive(&self) -> Result { + self.supports("supports_adaptive_thinking") + } +} + +/// Python's `value` in an error message: its `str()`. +pub fn python_str(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Bool(true) => "True".into(), + Value::Bool(false) => "False".into(), + Value::Null => "None".into(), + other => other.to_string(), + } +} + +/// `_apply_sampling_param`: the param, unless the model removed sampling params. +pub fn apply_sampling_param( + out: &mut Map, + caps: &ModelCapabilities<'_>, + drop: DropPolicy, + param: &str, + value: &Value, + output_key: &str, +) -> Result<(), Error> { + if caps.supports_sampling()? || (param == "temperature" && value.as_f64() == Some(1.0)) { + out.insert(output_key.to_owned(), value.clone()); + return Ok(()); + } + drop.unsupported(|| { + Error::InvalidRequest(ErrorDetail::UnsupportedParameter { + model: caps.model.clone(), + param: param.to_owned(), + value: python_str(value), + hint: match param { + "temperature" => "Only temperature=1 is supported. ".into(), + _ => String::new(), + }, + }) + }) +} + +/// `_valid_user_id`: Anthropic rejects an email or a phone number as `metadata.user_id`. +pub fn valid_user_id(user_id: &str) -> bool { + !(looks_like_email(user_id) || looks_like_phone(user_id)) +} + +fn looks_like_email(text: &str) -> bool { + let Some((local, domain)) = text.split_once('@') else { + return false; + }; + let local_ok = !local.is_empty() + && local + .chars() + .all(|c| c.is_ascii_alphanumeric() || "._%+-".contains(c)); + let domain_ok = domain + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-') + && domain.rsplit_once('.').is_some_and(|(head, tld)| { + !head.is_empty() && tld.len() >= 2 && tld.chars().all(|c| c.is_ascii_alphabetic()) + }); + local_ok && domain_ok +} + +fn looks_like_phone(text: &str) -> bool { + let body = text.strip_prefix('+').unwrap_or(text); + body.chars().count() >= 7 + && body + .chars() + .all(|c| c.is_numeric() || c.is_whitespace() || "()-".contains(c)) +} + +/// `_map_stop_sequences`: whitespace-only entries are dropped only under `drop_params`. +pub fn map_stop_sequences(stop: &Value, drop: DropPolicy) -> Option { + let keep = |text: &str| !(is_python_space(text) && drop.drops()); + match stop { + Value::String(text) => keep(text).then(|| json!([text])), + Value::Array(items) => { + let kept: Vec = items + .iter() + .filter(|item| item.as_str().is_none_or(keep)) + .cloned() + .collect(); + (!kept.is_empty()).then_some(Value::Array(kept)) + } + _ => None, + } +} + +/// `str.isspace()`: non-empty and all whitespace. +pub fn is_python_space(text: &str) -> bool { + !text.is_empty() && text.chars().all(char::is_whitespace) +} + +/// `sanitize_input_schema_for_anthropic`. +pub fn sanitize_input_schema(schema: Option<&Value>) -> Value { + let mut normalized = match schema { + Some(Value::Object(object)) => object.clone(), + _ => Map::new(), + }; + if normalized.get("type").and_then(Value::as_str) != Some("object") { + normalized.insert("type".into(), json!("object")); + } + normalized + .entry("properties") + .or_insert_with(|| Value::Object(Map::new())); + unpack_legacy_defs(&mut normalized); + Value::Object( + normalized + .into_iter() + .filter(|(key, _)| { + matches!( + key.as_str(), + "type" + | "properties" + | "additionalProperties" + | "required" + | "$defs" + | "strict" + ) + }) + .collect(), + ) +} + +fn invalid_tool(message: &str) -> Error { + unsupported_value(message.to_owned()) +} + +/// `_map_tool_helper` for the tool shapes chat translates: function and custom tools. +fn map_function_tool(tool: &Map) -> Result { + let function = tool.get("function").and_then(Value::as_object); + let Some(function) = function else { + return Err(invalid_tool("A function tool needs a `function` object.")); + }; + let name = function + .get("name") + .cloned() + .ok_or_else(|| invalid_tool("A function tool needs a name."))?; + let default_schema = json!({"type": "object", "properties": {}}); + let schema = sanitize_input_schema(Some(function.get("parameters").unwrap_or(&default_schema))); + let mut mapped = Map::new(); + mapped.insert("name".into(), name); + mapped.insert("input_schema".into(), schema); + mapped.insert("type".into(), json!("custom")); + let eager = match tool.get("eager_input_streaming") { + Some(Value::Bool(flag)) => Some(*flag), + Some(Value::Null) | None => match function.get("eager_input_streaming") { + Some(Value::Bool(flag)) => Some(*flag), + Some(Value::Null) | None => None, + Some(_) => return Err(invalid_tool("eager_input_streaming must be a boolean")), + }, + Some(_) => return Err(invalid_tool("eager_input_streaming must be a boolean")), + }; + if let Some(eager) = eager { + mapped.insert("eager_input_streaming".into(), json!(eager)); + } + if let Some(description) = function.get("description").filter(|d| !d.is_null()) { + mapped.insert("description".into(), description.clone()); + } + match (tool.get("cache_control"), function.get("cache_control")) { + (Some(cache_control), _) if !cache_control.is_null() => { + mapped.insert("cache_control".into(), cache_control.clone()); + } + (_, Some(Value::Object(cache_control))) => { + mapped.insert("cache_control".into(), Value::Object(cache_control.clone())); + } + _ => {} + } + match (tool.get("defer_loading"), function.get("defer_loading")) { + (Some(Value::Bool(flag)), _) => { + mapped.insert("defer_loading".into(), json!(flag)); + } + (Some(other), _) if !other.is_null() => { + return Err(invalid_tool("defer_loading must be a boolean")); + } + (_, Some(Value::Bool(flag))) => { + mapped.insert("defer_loading".into(), json!(flag)); + } + (_, Some(other)) if !other.is_null() => { + return Err(invalid_tool("defer_loading must be a boolean")); + } + _ => {} + } + let string_list = |value: &Value| { + value + .as_array() + .is_some_and(|items| items.iter().all(Value::is_string)) + }; + for key in ["allowed_callers", "input_examples"] { + let value = tool + .get(key) + .filter(|v| !v.is_null()) + .or_else(|| function.get(key).filter(|v| !v.is_null())); + if let Some(value) = value { + if key == "allowed_callers" && !string_list(value) { + return Err(invalid_tool("allowed_callers must be a list of strings")); + } + if key == "input_examples" && !value.is_array() { + return Err(invalid_tool("input_examples must be a list")); + } + mapped.insert(key.into(), value.clone()); + } + } + Ok(Value::Object(mapped)) +} + +/// `_map_tools` for function and custom tools and tools already in Anthropic's shape. +pub fn map_tools(tools: &Value) -> Result, Error> { + let Some(tools) = tools.as_array() else { + return Err(invalid_tool("tools must be a list")); + }; + let mut mapped = Vec::new(); + for tool in tools { + let Some(object) = tool.as_object() else { + return Err(invalid_tool("each tool must be an object")); + }; + if object.contains_key("input_schema") { + mapped.push(tool.clone()); + continue; + } + match object.get("type").and_then(Value::as_str) { + Some("function" | "custom") => mapped.push(map_function_tool(object)?), + Some("namespace") => { + for nested in object + .get("tools") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let Some(nested_object) = nested.as_object() else { + continue; + }; + if nested_object.contains_key("input_schema") { + mapped.push(nested.clone()); + } else if !nested_object.contains_key("function") + && nested_object.contains_key("name") + { + let kind = nested_object + .get("type") + .cloned() + .unwrap_or_else(|| json!("function")); + let function: Map = nested_object + .iter() + .filter(|(key, _)| key.as_str() != "type") + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let wrapped = Map::from_iter([ + ("type".to_owned(), kind.clone()), + ("function".to_owned(), Value::Object(function)), + ]); + match kind.as_str() { + Some("function" | "custom") => { + mapped.push(map_function_tool(&wrapped)?) + } + _ => { + return Err(Error::Unsupported("a hosted tool inside a namespace")); + } + } + } else if nested_object.contains_key("function") { + match nested_object.get("type").and_then(Value::as_str) { + Some("function" | "custom") => { + mapped.push(map_function_tool(nested_object)?) + } + _ => { + return Err(Error::Unsupported("a hosted tool inside a namespace")); + } + } + } + } + } + // Computer use, hosted tools, MCP servers and tool search each have their own + // shape and betas; they stay on Python's path. + _ => { + return Err(Error::Unsupported( + "a chat tool other than a function or custom tool", + )); + } + } + } + Ok(mapped) +} + +/// `_map_tool_choice`. +pub fn map_tool_choice(tool_choice: Option<&Value>, parallel: Option<&Value>) -> Option { + let mut choice: Option> = match tool_choice { + Some(Value::String(choice)) => match choice.as_str() { + "auto" => Some(json!({"type": "auto"})), + "required" => Some(json!({"type": "any"})), + "none" => Some(json!({"type": "none"})), + _ => None, + }, + Some(Value::Object(choice)) + if choice.contains_key("type") && !choice.contains_key("function") => + { + match choice.get("type").and_then(Value::as_str) { + Some("auto") => Some(json!({"type": "auto"})), + Some("required" | "any") => Some(json!({"type": "any"})), + Some("none") => Some(json!({"type": "none"})), + _ => None, + } + } + Some(Value::Object(choice)) => choice + .get("function") + .and_then(|function| function.get("name")) + .filter(|name| !name.is_null()) + .map(|name| json!({"type": "tool", "name": name})), + _ => None, + } + .and_then(|value| value.as_object().cloned()); + if let Some(parallel) = parallel.filter(|parallel| !parallel.is_null()) { + let disable = json!(!truthy(parallel)); + let choice_is_none = tool_choice.and_then(Value::as_str) == Some("none"); + match &mut choice { + _ if choice_is_none => {} + Some(choice) => { + choice.insert("disable_parallel_tool_use".into(), disable); + } + None => { + choice = Some(Map::from_iter([ + ("type".to_owned(), json!("auto")), + ("disable_parallel_tool_use".to_owned(), disable), + ])); + } + } + } + choice.map(Value::Object) +} + +/// `_apply_forced_tool_choice`: `any` / `tool` on a model that refuses forced tool use. +pub fn apply_forced_tool_choice( + choice: Value, + caps: &ModelCapabilities<'_>, + drop: DropPolicy, +) -> Result { + let kind = choice.get("type").and_then(Value::as_str); + if !matches!(kind, Some("any" | "tool")) || !caps.forced_tool_use_unsupported()? { + return Ok(choice); + } + drop.unsupported(|| { + unsupported_value(format!( + "{} does not support forced tool use (tool_choice='required' or a named tool). Use tool_choice='auto' and tell the model in the prompt when to call the tool, or set `litellm.drop_params = True` to downgrade to 'auto' automatically.", + caps.model + )) + })?; + let mut auto = Map::from_iter([("type".to_owned(), json!("auto"))]); + if let Some(disable) = choice.get("disable_parallel_tool_use") { + auto.insert("disable_parallel_tool_use".into(), disable.clone()); + } + Ok(Value::Object(auto)) +} + +fn enum_conflicts_with_declared_type(schema: &Map) -> bool { + let (Some(Value::Array(values)), Some(declared)) = (schema.get("enum"), schema.get("type")) + else { + return false; + }; + if declared.is_array() { + return true; + } + let check: fn(&Value) -> bool = match declared.as_str() { + Some("null") => Value::is_null, + Some("boolean") => Value::is_boolean, + Some("integer") => |v| v.is_i64() || v.is_u64(), + Some("number") => Value::is_number, + Some("string") => Value::is_string, + Some("array") => Value::is_array, + Some("object") => Value::is_object, + _ => return false, + }; + !values.iter().all(check) +} + +const CONSTRAINT_LABELS: [(&str, &str); 25] = [ + ("minItems", "minimum number of items: {}"), + ("maxItems", "maximum number of items: {}"), + ("uniqueItems", "all array items must be unique"), + ("contains", "array must contain an item matching: {}"), + ("minContains", "minimum number of matching items: {}"), + ("maxContains", "maximum number of matching items: {}"), + ("prefixItems", "leading items must match, in order: {}"), + ("minimum", "minimum value: {}"), + ("maximum", "maximum value: {}"), + ("exclusiveMinimum", "exclusive minimum value: {}"), + ("exclusiveMaximum", "exclusive maximum value: {}"), + ("multipleOf", "must be a multiple of {}"), + ("minLength", "minimum length: {}"), + ("maxLength", "maximum length: {}"), + ("minProperties", "minimum number of properties: {}"), + ("maxProperties", "maximum number of properties: {}"), + ( + "patternProperties", + "properties whose names match each pattern must satisfy: {}", + ), + ("propertyNames", "property names must satisfy: {}"), + ("dependentRequired", "dependent required properties: {}"), + ("dependentSchemas", "dependent schemas: {}"), + ( + "unevaluatedProperties", + "unevaluated properties must satisfy: {}", + ), + ("if", "conditional (if): {}"), + ("then", "conditional (then): {}"), + ("else", "conditional (else): {}"), + ("not", "must not match: {}"), +]; + +/// `filter_anthropic_output_schema`: the constraints structured output cannot enforce, +/// removed and described instead. +pub fn filter_output_schema(schema: &Value) -> Value { + let Some(object) = schema.as_object() else { + return schema.clone(); + }; + let notes: Vec = CONSTRAINT_LABELS + .iter() + .filter_map(|(field, label)| { + let value = object.get(*field)?; + if value == &Value::Bool(false) { + return None; + } + let rendered = match value { + Value::Object(_) | Value::Array(_) => { + crate::base_llm::chat::params::python_dumps(value) + } + other => python_str(other), + }; + Some(label.replace("{}", &rendered)) + }) + .collect(); + let mut result = Map::new(); + if !notes.is_empty() { + let note = format!("Note: {}.", notes.join(", ")); + let existing = object + .get("description") + .and_then(Value::as_str) + .unwrap_or_default(); + let description = match existing.is_empty() { + true => note, + false => format!("{existing} {note}"), + }; + result.insert("description".into(), json!(description)); + } + let drops_type = enum_conflicts_with_declared_type(object); + for (key, value) in object { + if CONSTRAINT_LABELS.iter().any(|(field, _)| field == key) + || (key == "type" && drops_type) + || (key == "description" && result.contains_key("description")) + { + continue; + } + match (key.as_str(), value) { + ("properties" | "$defs", Value::Object(nested)) => { + result.insert( + key.clone(), + Value::Object( + nested + .iter() + .map(|(k, v)| (k.clone(), filter_output_schema(v))) + .collect(), + ), + ); + } + ("items", Value::Object(_)) => { + result.insert(key.clone(), filter_output_schema(value)); + } + ("anyOf" | "oneOf", Value::Array(items)) => { + let mut any_of = match result.remove("anyOf") { + Some(Value::Array(existing)) => existing, + _ => Vec::new(), + }; + any_of.extend(items.iter().map(filter_output_schema)); + result.insert("anyOf".into(), Value::Array(any_of)); + } + ("allOf", Value::Array(items)) => { + result.insert( + key.clone(), + Value::Array(items.iter().map(filter_output_schema).collect()), + ); + } + _ => { + result.insert(key.clone(), value.clone()); + } + } + } + if result.get("type").and_then(Value::as_str) == Some("object") { + result.insert("additionalProperties".into(), json!(false)); + } + Value::Object(result) +} + +/// The schema a `response_format` names, if any. A `json_schema` without `schema` names none. +fn response_format_schema(value: &Map) -> Option { + if let Some(schema) = value.get("response_schema") { + return Some(schema.clone()).filter(|schema| !schema.is_null()); + } + value + .get("json_schema") + .and_then(|json_schema| json_schema.get("schema")) + .filter(|schema| !schema.is_null()) + .cloned() +} + +/// `map_response_format_to_anthropic_output_format`. +fn output_format(value: &Map) -> Option { + let mut schema = response_format_schema(value)?; + let mut defs = Map::new(); + if let Value::Object(object) = &mut schema { + let definitions = object.remove("definitions"); + let dollar_defs = object.remove("$defs"); + if let Some(Value::Object(found)) = dollar_defs.or(definitions) { + defs = found; + } + } + if !defs.is_empty() { + unpack_defs(&mut schema, &defs); + } + Some(json!({"type": "json_schema", "schema": filter_output_schema(&schema)})) +} + +/// `_create_json_tool_call_for_response_format`. +fn response_format_tool(schema: &Value) -> Value { + let mut input_schema = Map::from_iter([("type".to_owned(), json!("object"))]); + if let Value::Object(schema) = schema { + input_schema.extend(schema.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + json!({"name": RESPONSE_FORMAT_TOOL_NAME, "input_schema": input_schema}) +} + +/// `cap_thinking_budget_to_max_tokens`. +pub fn cap_thinking_budget(thinking: Value, max_tokens: Option<&Value>) -> Option { + let budget = thinking.get("budget_tokens").and_then(Value::as_i64); + let (Some(max_tokens), Some(budget)) = (max_tokens.and_then(Value::as_f64), budget) else { + return Some(thinking); + }; + if max_tokens <= MIN_THINKING_BUDGET_TOKENS as f64 { + return None; + } + if (budget as f64) < max_tokens { + return Some(thinking); + } + let kind = thinking + .get("type") + .cloned() + .unwrap_or_else(|| json!("enabled")); + Some(json!({"type": kind, "budget_tokens": (max_tokens - 1.0) as i64})) +} + +/// `_map_reasoning_effort`: `None` for `none`. +pub fn map_reasoning_effort( + effort: &str, + caps: &ModelCapabilities<'_>, +) -> Result, Error> { + if effort == "none" { + return Ok(None); + } + if caps.is_adaptive()? { + return Ok(Some(json!({"type": "adaptive", "display": "summarized"}))); + } + let budget = + reasoning_effort_budget(effort).ok_or_else(|| unmapped_reasoning_effort(effort))?; + Ok(Some(json!({"type": "enabled", "budget_tokens": budget}))) +} + +/// `_legacy_budget_to_effort`. +fn legacy_budget_to_effort( + budget: i64, + caps: &ModelCapabilities<'_>, +) -> Result<&'static str, Error> { + Ok( + if budget >= 8192 && caps.supports("supports_xhigh_reasoning_effort")? { + "xhigh" + } else if budget >= 4096 { + "high" + } else if budget >= 2048 { + "medium" + } else { + "low" + }, + ) +} + +/// `translate_legacy_thinking_for_adaptive_model`. +pub fn translate_legacy_thinking( + out: &mut Map, + caps: &ModelCapabilities<'_>, + drop: DropPolicy, +) -> Result<(), Error> { + if !caps.is_adaptive()? || caps.supports("supports_legacy_thinking")? { + return Ok(()); + } + let Some(Value::Object(thinking)) = out.get("thinking").cloned() else { + return Ok(()); + }; + if thinking.get("type").and_then(Value::as_str) != Some("enabled") { + return Ok(()); + } + let budget = match thinking.get("budget_tokens") { + Some(Value::Number(number)) => number.as_f64().unwrap_or(0.0) as i64, + Some(Value::String(text)) => text.trim().parse().unwrap_or(0), + _ => 0, + }; + let effort = legacy_budget_to_effort(budget, caps)?; + let mut rebuilt: Map = thinking + .iter() + .filter(|(key, _)| { + !matches!(key.as_str(), "type" | "budget_tokens" | "display") + && (key.as_str() == "block_binding" || !drop.drops()) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + rebuilt.insert("type".into(), json!("adaptive")); + if let Some(display) = thinking + .get("display") + .filter(|display| matches!(display.as_str(), Some("summarized" | "omitted"))) + { + rebuilt.insert("display".into(), display.clone()); + } + let mut output_config = Map::from_iter([("effort".to_owned(), json!(effort))]); + if let Some(Value::Object(existing)) = out.get("output_config") { + output_config.extend(existing.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + out.insert("thinking".into(), Value::Object(rebuilt)); + out.insert("output_config".into(), Value::Object(output_config)); + Ok(()) +} + +/// `web_search_options` as Anthropic's hosted web search tool. +pub fn web_search_tool(options: &Map) -> Value { + let mut tool = Map::from_iter([ + ("type".to_owned(), json!("web_search_20250305")), + ("name".to_owned(), json!("web_search")), + ]); + if let Some(location) = options.get("user_location").filter(|v| !v.is_null()) { + let mut user_location = Map::from_iter([("type".to_owned(), json!("approximate"))]); + if let Some(Value::Object(approximate)) = location.get("approximate") { + for (key, value) in approximate { + if matches!(key.as_str(), "city" | "region" | "country" | "timezone") { + user_location.insert(key.clone(), value.clone()); + } + } + } + tool.insert("user_location".into(), Value::Object(user_location)); + } + if let Some(size) = options.get("search_context_size").and_then(Value::as_str) { + let max_uses = match size { + "low" => Some(1), + "medium" => Some(5), + "high" => Some(10), + _ => None, + }; + if let Some(max_uses) = max_uses { + tool.insert("max_uses".into(), json!(max_uses)); + } + } + Value::Object(tool) +} + +/// `map_openai_context_management_to_anthropic`. +pub fn map_context_management(value: &Value) -> Option { + match value { + Value::Object(object) if object.contains_key("edits") => Some(value.clone()), + Value::Array(entries) => { + let edits: Vec = entries + .iter() + .filter_map(Value::as_object) + .filter(|entry| entry.get("type").and_then(Value::as_str) == Some("compaction")) + .map(|entry| { + let mut edit = Map::from_iter([("type".to_owned(), json!("compact_20260112"))]); + if let Some(threshold) = entry.get("compact_threshold").and_then(Value::as_f64) + { + edit.insert( + "trigger".into(), + json!({"type": "input_tokens", "value": threshold as i64}), + ); + } + for (key, value) in entry { + if !matches!(key.as_str(), "type" | "compact_threshold") { + edit.insert(key.clone(), value.clone()); + } + } + Value::Object(edit) + }) + .collect(); + (!edits.is_empty()).then(|| json!({"edits": edits})) + } + _ => None, + } +} + +/// `max_tokens` as Python keeps it: an int as sent, any other number rounded to at least 1. +pub fn max_tokens_value(value: &Value) -> Result { + match value { + Value::Number(number) if number.is_i64() || number.is_u64() => Ok(value.clone()), + Value::Number(number) => { + let rounded = number.as_f64().unwrap_or(1.0).round_ties_even(); + Ok(json!((rounded as i64).max(1))) + } + Value::Bool(flag) => Ok(json!(i64::from(*flag))), + other => Err(Error::InvalidRequest(ErrorDetail::InvalidValue { + field: "max_tokens", + expected: "a number", + actual: other.clone(), + })), + } +} + +/// What `map_openai_params` hands to `transform_request`. +#[derive(Debug, Default)] +pub struct MappedParams { + pub params: Map, + pub json_mode: bool, +} + +/// `get_supported_openai_params`. +pub fn supported_params(caps: &ModelCapabilities<'_>) -> Result, Error> { + let mut params = vec![ + "stream", + "stop", + "temperature", + "top_p", + "max_tokens", + "max_completion_tokens", + "tools", + "tool_choice", + "extra_headers", + "parallel_tool_calls", + "response_format", + "user", + "web_search_options", + "speed", + "context_management", + "cache_control", + ]; + let mut reasoning_keys = caps.candidates.clone(); + reasoning_keys.insert(1, format!("anthropic/{}", caps.model)); + if caps.model.contains("claude-3-7-sonnet") + || caps.is_adaptive()? + || capability(&caps.context, &reasoning_keys, "supports_reasoning")? == Some(true) + { + params.extend(["thinking", "reasoning_effort"]); + } + params.push("service_tier"); + Ok(params) +} + +/// `AnthropicConfig.map_openai_params`. +pub fn map_openai_params( + openai: &Map, + caps: &ModelCapabilities<'_>, + drop: DropPolicy, +) -> Result { + let thinking_enabled = crate::base_llm::chat::params::is_thinking_enabled(openai); + let mut out = Map::new(); + let mut json_mode = false; + let add_tools = |out: &mut Map, tools: Vec| match out.get_mut("tools") { + Some(Value::Array(existing)) => existing.extend(tools), + _ => { + out.insert("tools".into(), Value::Array(tools)); + } + }; + for (name, value) in ordered(openai) { + match name { + "max_tokens" | "max_completion_tokens" => { + out.insert("max_tokens".into(), max_tokens_value(value)?); + } + "tools" => add_tools(&mut out, map_tools(value)?), + "tool_choice" | "parallel_tool_calls" => { + if let Some(choice) = + map_tool_choice(openai.get("tool_choice"), openai.get("parallel_tool_calls")) + { + out.insert( + "tool_choice".into(), + apply_forced_tool_choice(choice, caps, drop)?, + ); + } + } + "stream" if value == &Value::Bool(true) => { + out.insert("stream".into(), json!(true)); + } + "stop" if value.is_string() || value.is_array() => { + if let Some(stop) = map_stop_sequences(value, drop) { + out.insert("stop_sequences".into(), stop); + } + } + "temperature" | "top_p" => { + apply_sampling_param(&mut out, caps, drop, name, value, name)? + } + "response_format" => { + let Some(format) = value.as_object() else { + continue; + }; + if caps.supports("supports_native_structured_output")? { + if let Some(output_format) = output_format(format) { + out.insert("output_format".into(), output_format); + } + } else { + if format.get("type").and_then(Value::as_str) == Some("text") { + continue; + } + let Some(schema) = response_format_schema(format) else { + continue; + }; + if !thinking_enabled && !caps.forced_tool_use_unsupported()? { + out.insert( + "tool_choice".into(), + json!({"name": RESPONSE_FORMAT_TOOL_NAME, "type": "tool"}), + ); + } + add_tools(&mut out, vec![response_format_tool(&schema)]); + } + json_mode = true; + } + "user" => { + if let Some(user) = value.as_str().filter(|user| valid_user_id(user)) { + out.insert("metadata".into(), json!({"user_id": user})); + } + } + "thinking" => { + let adaptive_request = + value.get("type").and_then(Value::as_str) == Some("adaptive"); + if value.is_object() && adaptive_request && !caps.is_adaptive()? { + let max_tokens = openai + .get("max_completion_tokens") + .filter(|v| truthy(v)) + .or_else(|| openai.get("max_tokens")) + .filter(|v| !v.is_null()); + let legacy = map_reasoning_effort("medium", caps)?; + match legacy.and_then(|legacy| cap_thinking_budget(legacy, max_tokens)) { + Some(capped) => { + out.insert("thinking".into(), capped); + } + None => { + out.remove("thinking"); + } + } + } else { + out.insert("thinking".into(), value.clone()); + translate_legacy_thinking(&mut out, caps, drop)?; + } + } + "reasoning_effort" => { + let effort = match value { + Value::Object(effort) => effort.get("effort"), + other => Some(other), + }; + let Some(effort) = effort.and_then(Value::as_str) else { + continue; + }; + match map_reasoning_effort(effort, caps)? { + None => { + out.remove("thinking"); + out.remove("output_config"); + } + Some(thinking) => { + out.insert("thinking".into(), thinking); + if caps.is_adaptive()? { + let mapped = output_config_effort(effort) + .ok_or_else(|| unmapped_reasoning_effort(effort))?; + out.insert("output_config".into(), json!({"effort": mapped})); + } + } + } + } + "web_search_options" => { + if let Some(options) = value.as_object() { + add_tools(&mut out, vec![web_search_tool(options)]); + } + } + "context_management" => { + if let Some(mapped) = map_context_management(value) { + out.insert("context_management".into(), mapped); + } + } + "service_tier" => { + let Some(tier) = value.as_str() else { continue }; + match tier { + "auto" => { + out.insert("service_tier".into(), json!("auto")); + } + "default" | "standard_only" => { + out.insert("service_tier".into(), json!("standard_only")); + } + other => drop.unsupported(|| { + unsupported_value(format!( + "{} does not support service_tier='{other}'. Anthropic accepts 'auto', 'standard_only', or OpenAI's 'default' (sent as 'standard_only'). To drop unsupported params, set `litellm.drop_params = True`.", + caps.model + )) + })?, + } + } + _ => {} + } + } + // `update_optional_params_with_thinking_tokens`. + let thinking_on = out + .get("thinking") + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("enabled"); + if thinking_on + && !openai.contains_key("max_tokens") + && !openai.contains_key("max_completion_tokens") + && let Some(budget) = out + .get("thinking") + .and_then(|thinking| thinking.get("budget_tokens")) + .and_then(Value::as_i64) + { + out.insert( + "max_tokens".into(), + json!(budget + DEFAULT_MAX_TOKENS as i64), + ); + } + Ok(MappedParams { + params: out, + json_mode, + }) +} + +/// The schema of the request's `json_tool_call` tool (`JsonModeRequest.tool_schema`). +pub fn json_tool_schema(tools: Option<&Value>) -> Option { + tools? + .as_array()? + .iter() + .find(|tool| tool.get("name").and_then(Value::as_str) == Some(RESPONSE_FORMAT_TOOL_NAME)) + .and_then(|tool| tool.get("input_schema")) + .and_then(ResponseFormatSchema::from_schema) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn object(value: Value) -> Map { + value.as_object().unwrap().clone() + } + + fn caps(model: &str) -> ModelCapabilities<'static> { + ModelCapabilities::new( + model, + &ChatRequestContext { + drop_params: Some(false), + model_info: None, + }, + ) + } + + fn map(params: Value) -> MappedParams { + map_openai_params( + &object(params), + &caps("claude-sonnet-4-5"), + DropPolicy::new(Some(false)), + ) + .unwrap() + } + + #[rstest] + fn candidates_cover_bedrock_ids_and_dotted_versions() { + let candidates = model_map_candidates("us.anthropic.claude-sonnet-4.6-20251101-v1:0"); + assert!(candidates.contains(&"anthropic.claude-sonnet-4.6-20251101-v1:0".to_string())); + assert!(candidates.contains(&"us.anthropic.claude-sonnet-4-6".to_string())); + assert!( + candidates.contains(&"bedrock/anthropic.claude-sonnet-4.6-20251101-v1:0".to_string()) + ); + assert!(model_map_candidates("claude-x@20250101").contains(&"claude-x".to_string())); + } + + #[rstest] + fn tools_and_choices_map_like_python() { + let mapped = map(json!({ + "tools": [{"type": "function", "function": {"name": "f", "description": "d", "parameters": {"type": "object", "properties": {"a": {}}, "title": "x"}}, "cache_control": {"type": "ephemeral"}}], + "tool_choice": "required", + "parallel_tool_calls": false + })); + assert_eq!( + mapped.params, + object(json!({ + "tools": [{"name": "f", "input_schema": {"type": "object", "properties": {"a": {}}}, "type": "custom", "description": "d", "cache_control": {"type": "ephemeral"}}], + "tool_choice": {"type": "any", "disable_parallel_tool_use": true} + })) + ); + assert_eq!( + map(json!({"tool_choice": {"type": "function", "function": {"name": "f"}}})).params["tool_choice"], + json!({"type": "tool", "name": "f"}) + ); + assert_eq!( + map(json!({"parallel_tool_calls": true})).params["tool_choice"], + json!({"type": "auto", "disable_parallel_tool_use": false}) + ); + } + + #[rstest] + fn max_tokens_stop_user_and_sampling() { + let mapped = map( + json!({"max_completion_tokens": 10.6, "stop": "END", "user": "u-1", "temperature": 0.2, "top_p": 0.5}), + ); + assert_eq!( + mapped.params, + object( + json!({"temperature": 0.2, "top_p": 0.5, "stop_sequences": ["END"], "max_tokens": 11, "metadata": {"user_id": "u-1"}}) + ) + ); + assert!( + !map(json!({"user": "a@b.co"})) + .params + .contains_key("metadata") + ); + assert!( + !map(json!({"user": "+1 (555) 123-4567"})) + .params + .contains_key("metadata") + ); + } + + #[rstest] + fn removed_sampling_params_are_refused_or_dropped() { + let refused = map_openai_params( + &object(json!({"temperature": 0.3})), + &caps("claude-opus-4-7"), + DropPolicy::new(Some(false)), + ) + .unwrap_err(); + assert_eq!( + refused.to_string(), + "invalid request: claude-opus-4-7 does not support temperature=0.3. Only temperature=1 is supported. To drop unsupported params, set `litellm.drop_params = True`." + ); + let dropped = map_openai_params( + &object(json!({"temperature": 0.3, "top_p": 0.1})), + &caps("claude-opus-4-7"), + DropPolicy::new(Some(true)), + ) + .unwrap(); + assert!(dropped.params.is_empty()); + } + + #[rstest] + fn response_format_becomes_a_forced_tool() { + let mapped = map( + json!({"response_format": {"type": "json_schema", "json_schema": {"name": "x", "schema": {"type": "object", "properties": {"a": {"type": "string"}}}}}}), + ); + assert!(mapped.json_mode); + assert_eq!( + mapped.params, + object(json!({ + "tool_choice": {"name": "json_tool_call", "type": "tool"}, + "tools": [{"name": "json_tool_call", "input_schema": {"type": "object", "properties": {"a": {"type": "string"}}}}] + })) + ); + assert!(!map(json!({"response_format": {"type": "text"}})).json_mode); + } + + #[rstest] + fn reasoning_effort_sets_a_budget_and_max_tokens() { + let mapped = map(json!({"reasoning_effort": "low"})); + assert_eq!( + mapped.params, + object( + json!({"thinking": {"type": "enabled", "budget_tokens": 1024}, "max_tokens": 5120}) + ) + ); + assert!( + map_openai_params( + &object(json!({"reasoning_effort": "huge"})), + &caps("claude-sonnet-4-5"), + DropPolicy::new(Some(false)) + ) + .is_err() + ); + } + + #[rstest] + fn output_schemas_lose_unenforceable_constraints() { + assert_eq!( + filter_output_schema( + &json!({"type": "object", "properties": {"n": {"type": "integer", "minimum": 1, "description": "count"}, "o": {"oneOf": [{"type": "string"}]}}}) + ), + json!({"type": "object", "properties": {"n": {"description": "count Note: minimum value: 1.", "type": "integer"}, "o": {"anyOf": [{"type": "string"}]}}, "additionalProperties": false}) + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs index 49a81adc32e..bbfa16cc344 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs @@ -1,11 +1,15 @@ +//! Python's `AnthropicConfig` for `/chat/completions`: `get_optional_params` and +//! `transform_request` on the way in, `transform_parsed_response` on the way out. + +use std::collections::{BTreeSet, HashMap, HashSet}; + use litellm_auth::SecretValue; -use litellm_core_utils::{ - core_helpers::{finish_reason_for, unix_now, usage_from_parts}, - prompt_templates::factory::{Conversation, build_conversation}, -}; +use litellm_core_utils::core_helpers::{finish_reason_for, unix_now, usage_from_parts}; use litellm_llms_types::{ formats::chat_completions::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, ChatCompletionsChoice, ChatCompletionsChoiceMessage, + ChatCompletionsResponse, ChatMessage, }, providers::anthropic::DEFAULT_HEADERS, }; @@ -13,9 +17,19 @@ use serde::Deserialize; use serde_json::{Map, Value, json}; use crate::{ - Error, + Error, ErrorDetail, anthropic::{ - chat::handler::ModelResponseIterator, + chat::{ + handler::ModelResponseIterator, + messages::{ + anthropic_messages_pt, place_mid_conversation_system, role, + split_leading_system_run, system_blocks, + }, + params::{ + MappedParams, ModelCapabilities, apply_sampling_param, json_tool_schema, + map_openai_params, map_tools, supported_params, valid_user_id, + }, + }, common_utils::{ API_KEY_PLACEMENT, complete_anthropic_url, forwarded_oauth_bearer, resolve_anthropic_api_key, @@ -24,50 +38,39 @@ use crate::{ base_llm::{ auth::AuthScheme, chat::{ + media::{data_url, remote_urls}, + params::{ + ChatRequestContext, DEFAULT_MAX_TOKENS, RESPONSE_FORMAT_TOOL_NAME, capability, + python_dumps, sort_params, + }, + response_format::json_tool_answer, streaming::{ChatStream, StreamShape}, transformation::{ BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, - STREAM_PARAM, Unsupported, ValidatedEnvironment, unsupported_message, - unsupported_param, + STREAM_PARAM, Unsupported, ValidatedEnvironment, }, }, messages::streaming::event_stream as messages_event_stream, }, }; -/// Anthropic parameter names, post `map_openai_params`, that the Rust path can -/// place verbatim in the Messages body. -/// -/// `top_k` is deliberately absent even though the Messages API takes it. -/// `temperature` and `top_p` reach this gate already resolved, because -/// `map_openai_params` runs first and applies `_apply_sampling_param` to them. -/// `top_k` bypasses `map_openai_params` entirely, so Python applies that same -/// per-model gate inside `transform_request`, the function this route replaces. -/// Forwarding it would send `top_k` to a model that removed sampling params and -/// take a 400 after the call, where Python drops it and succeeds. -const SUPPORTED_PARAMS: &[(&str, &str)] = &[ - ("max_tokens", "max_tokens"), - ("temperature", "temperature"), - ("top_p", "top_p"), - ("stop", "stop_sequences"), -]; +pub const BETA_HEADER: &str = "anthropic-beta"; +const TOOL_NAME_MAX_LEN: usize = 128; +const VALID_EFFORTS: [&str; 5] = ["high", "medium", "low", "xhigh", "max"]; #[derive(Deserialize)] -struct TextResponseProjection { +struct ResponseProjection { model: String, - content: Vec, + content: Vec, usage: ResponseUsageProjection, + #[serde(default)] stop_reason: Option, -} - -#[derive(Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum TextResponseBlock { - Text { - text: String, - }, - #[serde(other)] - Other, + #[serde(default)] + stop_details: Option, + #[serde(default)] + context_management: Option, + #[serde(default)] + container: Option, } #[derive(Deserialize)] @@ -84,6 +87,705 @@ pub struct AnthropicConfig; pub const ANTHROPIC_CHAT_COMPLETIONS_CONFIG: AnthropicConfig = AnthropicConfig; +/// `add_dummy_tool`, the tool Anthropic needs when the history holds tool calls and the +/// request names no tools. +fn dummy_tool() -> Value { + json!([{ + "type": "function", + "function": { + "name": "dummy_tool", + "description": "This is a dummy tool call", + "parameters": {"type": "object", "properties": {}}, + }, + }]) +} + +/// `get_prefix_prompt`: the text of a trailing `{"role": "assistant", "prefix": true}`. +fn prefix_prompt(messages: &[Value]) -> Option { + let last = messages.last()?; + let is_prefix = last + .get("prefix") + .is_some_and(crate::base_llm::chat::params::truthy); + (last.get("role").and_then(Value::as_str) == Some("assistant") && is_prefix) + .then(|| { + last.get("content") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .flatten() +} + +/// `_basic_sanitize_anthropic_tool_name`. +fn basic_sanitize_tool_name(name: &str) -> String { + name.chars() + .map( + |c| match c.is_ascii_alphanumeric() || c == '_' || c == '-' { + true => c, + false => '_', + }, + ) + .take(TOOL_NAME_MAX_LEN) + .collect() +} + +/// `_build_anthropic_tool_name_maps`: `(forward, reverse)` for the names that change. +fn tool_name_maps(originals: &[String]) -> (HashMap, HashMap) { + let mut forward: HashMap = HashMap::new(); + let mut used: HashSet = originals + .iter() + .filter(|original| basic_sanitize_tool_name(original) == **original) + .cloned() + .collect(); + for original in originals { + let candidate = basic_sanitize_tool_name(original); + if candidate == *original || forward.contains_key(original) { + continue; + } + let mut unique = candidate.clone(); + let mut n = 1; + while used.contains(&unique) { + n += 1; + let suffix = format!("_{n}"); + let head: String = candidate + .chars() + .take(TOOL_NAME_MAX_LEN - suffix.len()) + .collect(); + unique = format!("{head}{suffix}"); + } + used.insert(unique.clone()); + forward.insert(original.clone(), unique); + } + let reverse = forward + .iter() + .map(|(original, sanitized)| (sanitized.clone(), original.clone())) + .collect(); + (forward, reverse) +} + +/// `_sanitize_tool_names_in_request` and `_rewrite_tool_names_in_messages`. +fn sanitize_tool_names( + params: &mut Map, + messages: &mut [Value], +) -> HashMap { + let Some(Value::Array(tools)) = params.get_mut("tools") else { + return HashMap::new(); + }; + let originals: Vec = tools + .iter() + .filter(|tool| tool.get("type").and_then(Value::as_str) == Some("custom")) + .filter_map(|tool| tool.get("name").and_then(Value::as_str)) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + .collect(); + let (forward, reverse) = tool_name_maps(&originals); + if forward.is_empty() { + return reverse; + } + let rename = |value: Option<&mut Value>| { + if let Some(Value::String(name)) = value + && let Some(sanitized) = forward.get(name.as_str()) + { + *name = sanitized.clone(); + } + }; + for tool in tools.iter_mut() { + if tool.get("type").and_then(Value::as_str) == Some("custom") { + rename(tool.get_mut("name")); + } + } + if let Some(choice) = params.get_mut("tool_choice") + && choice.get("type").and_then(Value::as_str) == Some("tool") + { + rename(choice.get_mut("name")); + } + for message in messages.iter_mut() { + if let Some(Value::Array(calls)) = message.get_mut("tool_calls") { + for call in calls { + rename(call.get_mut("function").and_then(|f| f.get_mut("name"))); + } + } + rename( + message + .get_mut("function_call") + .and_then(|f| f.get_mut("name")), + ); + } + reverse +} + +/// The `anthropic-beta` values the request's features need: the ones Python's +/// `validate_environment` collects and those `update_headers_with_optional_anthropic_beta` +/// adds. Python joins the first lot from a set, so their order carries no meaning. +fn request_betas( + params: &Map, + original_messages: &[Value], + anthropic_messages: &[Value], + caps: &ModelCapabilities<'_>, +) -> Result, Error> { + let tools: Vec<&Value> = params + .get("tools") + .and_then(Value::as_array) + .into_iter() + .flatten() + .collect(); + let mut environment: BTreeSet<&'static str> = BTreeSet::new(); + let tool_field = |tool: &Value, key: &str| -> Option { + tool.get(key) + .or_else(|| tool.get("function").and_then(|f| f.get(key))) + .cloned() + }; + if params + .get("mcp_servers") + .is_some_and(crate::base_llm::chat::params::truthy) + { + environment.insert("mcp-client-2025-04-04"); + } + if tools.iter().any(|tool| { + tool_field(tool, "allowed_callers") + .and_then(|callers| callers.as_array().cloned()) + .is_some_and(|callers| callers.iter().any(|c| c == "code_execution_20250825")) + || tool_field(tool, "input_examples") + .and_then(|examples| examples.as_array().map(|e| !e.is_empty())) + .unwrap_or(false) + }) { + environment.insert("advanced-tool-use-2025-11-20"); + } + let output_config = params.get("output_config"); + if output_config + .and_then(|config| config.get("effort")) + .and_then(Value::as_str) + .is_some_and(|effort| !effort.is_empty()) + && !caps.is_adaptive()? + { + environment.insert("effort-2025-11-24"); + } + if original_messages + .iter() + .any(|message| message.get("output_config").is_some()) + { + environment.insert("mid-conversation-output-config-2026-07-01"); + } + let thinking = params.get("thinking"); + if thinking + .and_then(|t| t.get("type")) + .and_then(Value::as_str) + .is_some_and(|kind| matches!(kind, "adaptive" | "enabled")) + && thinking + .and_then(|t| t.get("display")) + .and_then(Value::as_str) + == Some("updates") + { + environment.insert("thinking-display-updates-2026-08-18"); + } + if original_messages.iter().any(|message| { + role(message) == "system" + && message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|block| { + matches!( + block.get("type").and_then(Value::as_str), + Some("tool_addition" | "tool_removal") + ) && block + .get("tool") + .and_then(|t| t.get("type")) + .and_then(Value::as_str) + == Some("tool_reference") + }) + }) { + environment.insert("mid-conversation-tool-changes-2026-07-01"); + } + + let mut betas: Vec = environment.into_iter().map(str::to_owned).collect(); + let mut add = |beta: &str| { + if !betas.iter().any(|existing| existing == beta) { + betas.push(beta.to_owned()); + } + }; + let signed_compaction = anthropic_messages.iter().any(|message| { + message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|block| { + block.get("type").and_then(Value::as_str) == Some("compaction") + && block + .get("signature") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()) + }) + }); + if params.get("compaction").is_some_and(|v| !v.is_null()) || signed_compaction { + add("compact-2026-09-04"); + } + if let Some(context_management) = params.get("context_management").filter(|v| !v.is_null()) { + let edits = match context_management { + Value::Object(object) if object.contains_key("edits") => { + object.get("edits").and_then(Value::as_array).cloned() + } + Value::Array(entries) => Some(entries.clone()), + _ => None, + }; + if let Some(edits) = edits { + let compact = |edit: &Value| { + matches!( + edit.get("type").and_then(Value::as_str), + Some("compact_20260112" | "compaction") + ) + }; + if edits.iter().any(compact) { + add("compact-2026-01-12"); + } + if edits.iter().any(|edit| !compact(edit)) { + add("context-management-2025-06-27"); + } + } + } + if params.get("output_format").is_some_and(|v| !v.is_null()) + || output_config + .and_then(|config| config.get("format")) + .is_some_and(|v| !v.is_null()) + { + add("structured-outputs-2025-11-13"); + } + if params.get("speed").and_then(Value::as_str) == Some("fast") { + add("fast-mode-2026-02-01"); + } + Ok(betas) +} + +/// `_apply_output_config`: the effort check Python makes before sending `output_config`. +fn apply_output_config( + body: &mut Map, + caps: &ModelCapabilities<'_>, + drops: bool, +) -> Result<(), Error> { + let Some(Value::Object(output_config)) = body.get("output_config").cloned() else { + return Ok(()); + }; + if output_config.is_empty() { + return Ok(()); + } + let supports_effort = || -> Result { + if caps.supports("supports_output_config")? { + return Ok(true); + } + for level in ["low", "minimal", "medium", "high", "xhigh", "max"] { + if caps.supports(&format!("supports_{level}_reasoning_effort"))? { + return Ok(true); + } + } + Ok(false) + }; + if drops && output_config.keys().any(|key| key != "format") && !supports_effort()? { + match output_config.get("format") { + None | Some(Value::Null) => { + body.remove("output_config"); + } + Some(format) => { + body.insert("output_config".into(), json!({"format": format})); + } + } + return Ok(()); + } + let effort = output_config + .get("effort") + .filter(|effort| !effort.is_null()); + if let Some(effort) = effort { + let text = effort.as_str().filter(|text| VALID_EFFORTS.contains(text)); + if text.is_none() { + let shown = match effort { + Value::String(text) => format!("'{text}'"), + other => python_dumps(other), + }; + return Err(Error::InvalidRequest(ErrorDetail::Message(format!( + "Invalid effort value: {shown}. Must be one of: 'high', 'medium', 'low', 'xhigh', 'max'" + )))); + } + let value = match text { + Some("max") + if !(caps.is_adaptive()? || caps.supports("supports_max_reasoning_effort")?) => + { + Some("max") + } + Some("xhigh") if !caps.supports("supports_xhigh_reasoning_effort")? => Some("xhigh"), + _ => None, + }; + if let Some(value) = value { + return Err(Error::InvalidRequest(ErrorDetail::UnsupportedValue { + field: "effort", + value, + model: caps.model.clone(), + })); + } + } + Ok(()) +} + +/// `get_max_tokens_for_model`: the exact cost-map entry's limit, else 4096. +fn default_max_tokens(model: &str, context: &ChatRequestContext<'_>) -> Result { + let Some(info) = context.model_info else { + return match context.drop_params { + None => Err(Error::Unsupported( + "a request whose default max_tokens comes from the model cost map", + )), + Some(_) => Ok(json!(DEFAULT_MAX_TOKENS)), + }; + }; + let limit = info + .entry(model) + .and_then(|entry| match entry.get("max_output_tokens") { + Some(value) => Some(value.clone()), + None => entry.get("max_tokens").cloned(), + }); + Ok(match limit { + Some(value) if value.is_number() => value, + _ => json!(DEFAULT_MAX_TOKENS), + }) +} + +impl AnthropicConfig { + fn build_request( + &self, + model: &str, + messages: Vec, + optional_params: Map, + context: &ChatRequestContext<'_>, + ) -> Result { + let caps = ModelCapabilities::new(model, context); + let request_metadata = optional_params.get("metadata").cloned(); + let sorted = sort_params( + optional_params, + &[], + context, + |_| supported_params(&caps), + "anthropic", + model, + )?; + let drop = sorted.drop; + let MappedParams { + mut params, + json_mode, + } = map_openai_params(&sorted.openai, &caps, drop)?; + for name in &sorted.allowed { + if let Some(value) = sorted.openai.get(name) + && !params.contains_key(name) + { + params.insert(name.clone(), value.clone()); + } + } + params.extend(sorted.provider); + + let mut messages: Vec = messages + .iter() + .map(serde_json::to_value) + .collect::>() + .map_err(|error| Error::InvalidRequest(ErrorDetail::invalid("messages", error)))?; + let original_messages = messages.clone(); + let prefix_prompt = prefix_prompt(&messages); + + if !params.contains_key("tools") + && messages.iter().any(|message| { + message + .get("tool_calls") + .is_some_and(|calls| !calls.is_null()) + }) + { + params.insert("tools".into(), Value::Array(map_tools(&dummy_tool())?)); + } + + // `_maybe_drop_speed_param`: fast mode is the Anthropic API's own, per exact entry. + if let Some(speed) = params.get("speed").cloned() + && capability(context, &[model.to_owned()], "supports_speed")? != Some(true) + { + drop.unsupported(|| { + let shown = match &speed { + Value::String(text) => format!("'{text}'"), + other => python_dumps(other), + }; + Error::InvalidRequest(ErrorDetail::Message(format!( + "{model} does not support speed={shown}. To drop unsupported params, set `litellm.drop_params = True`." + ))) + })?; + params.remove("speed"); + } + + // `maybe_drop_disabled_thinking`. + if params + .get("thinking") + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("disabled") + && caps.supports("thinking_always_on")? + { + params.remove("thinking"); + } + + let tool_name_reverse_map = sanitize_tool_names(&mut params, &mut messages); + + let (leading, later) = split_leading_system_run(messages); + let system = system_blocks(&leading); + if !system.is_empty() { + params.insert("system".into(), Value::Array(system)); + } + let anthropic_messages = anthropic_messages_pt(place_mid_conversation_system(later))?; + + let betas = request_betas(¶ms, &original_messages, &anthropic_messages, &caps)?; + + if !params.contains_key("max_tokens") { + params.insert("max_tokens".into(), default_max_tokens(model, context)?); + } + + if let Some(user_id) = request_metadata + .as_ref() + .and_then(|metadata| metadata.get("user_id")) + .and_then(Value::as_str) + .filter(|user_id| valid_user_id(user_id)) + { + params.insert("metadata".into(), json!({"user_id": user_id})); + } + if let Some(Value::Object(metadata)) = params.get("metadata") { + match metadata.get("user_id").filter(|user_id| !user_id.is_null()) { + Some(user_id) => { + let reduced = json!({"user_id": user_id}); + params.insert("metadata".into(), reduced); + } + None => { + params.remove("metadata"); + } + } + } + params.remove("is_vertex_request"); + params.remove("client_metadata"); + params.remove("json_mode"); + + if let Some(top_k) = params.remove("top_k").filter(|top_k| !top_k.is_null()) { + apply_sampling_param(&mut params, &caps, drop, "top_k", &top_k, "top_k")?; + } + + let speed = params + .get("speed") + .and_then(Value::as_str) + .map(str::to_owned); + let response_format_schema = json_mode + .then(|| json_tool_schema(params.get("tools"))) + .flatten(); + + let mut body = Map::from_iter([ + ("model".to_owned(), json!(model)), + ("messages".to_owned(), Value::Array(anthropic_messages)), + ]); + body.extend(params); + apply_output_config(&mut body, &caps, drop.drops())?; + + Ok(ProviderChatRequestData { + body: Value::Object(body), + stream_shape: StreamShape { + json_mode, + speed, + tool_name_reverse_map, + response_format_schema, + prefix_prompt, + }, + headers: match betas.is_empty() { + true => Vec::new(), + false => vec![(BETA_HEADER.to_owned(), betas.join(", "))], + }, + }) + } +} + +/// `convert_tool_use_to_openai_format`, the name put back the way the caller spelled it. +fn tool_call(block: &Value, index: usize, shape: &StreamShape) -> ChatCompletionToolCallChunk { + let name = block.get("name").and_then(Value::as_str).map(|name| { + shape + .tool_name_reverse_map + .get(name) + .cloned() + .unwrap_or_else(|| name.to_owned()) + }); + ChatCompletionToolCallChunk { + id: block.get("id").and_then(Value::as_str).map(str::to_owned), + tool_type: "function".into(), + function: ChatCompletionToolCallFunctionChunk { + name, + arguments: python_dumps(block.get("input").unwrap_or(&Value::Null)), + provider_specific_fields: None, + }, + index: index as i64, + } +} + +/// Python's `refusal_provider_specific_fields`. +fn refusal_fields(stop_reason: Option<&str>, stop_details: Option<&Value>) -> Map { + let mut fields = Map::new(); + match stop_details.filter(|details| !details.is_null()) { + Some(details) => { + fields.insert("stop_reason".into(), json!(stop_reason)); + fields.insert("stop_details".into(), details.clone()); + } + None if stop_reason == Some("refusal") => { + fields.insert("stop_reason".into(), json!("refusal")); + } + None => {} + } + fields +} + +fn anthropic_response( + body: ResponseProjection, + shape: &StreamShape, +) -> Result { + let mut text = String::new(); + let mut citations: Option> = None; + let mut thinking_blocks: Option> = None; + let mut tool_calls: Vec = Vec::new(); + let mut web_search_results: Option> = None; + let mut tool_results: Option> = None; + let mut compaction_blocks: Option> = None; + for (index, block) in body.content.iter().enumerate() { + let kind = block + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + match kind { + "text" => text.push_str( + block + .get("text") + .and_then(Value::as_str) + .unwrap_or_default(), + ), + "tool_use" | "server_tool_use" => tool_calls.push(tool_call(block, index, shape)), + result if result.ends_with("_tool_result") => { + let list = match result { + "web_search_tool_result" | "web_fetch_tool_result" => &mut web_search_results, + _ => &mut tool_results, + }; + list.get_or_insert_with(Vec::new).push(block.clone()); + } + "thinking" | "redacted_thinking" => thinking_blocks + .get_or_insert_with(Vec::new) + .push(block.clone()), + "compaction" => compaction_blocks + .get_or_insert_with(Vec::new) + .push(block.clone()), + _ => {} + } + if let Some(Value::Array(block_citations)) = block.get("citations") { + let supported_text = block.get("text").cloned().unwrap_or_else(|| json!("")); + citations.get_or_insert_with(Vec::new).push(Value::Array( + block_citations + .iter() + .map(|citation| { + let mut citation = citation.as_object().cloned().unwrap_or_default(); + citation.insert("supported_text".into(), supported_text.clone()); + Value::Object(citation) + }) + .collect(), + )); + } + } + let reasoning_content = thinking_blocks.as_ref().map(|blocks| { + blocks + .iter() + .filter_map(|block| block.get("thinking").and_then(Value::as_str)) + .collect::() + }); + if let Some(prefix) = &shape.prefix_prompt + && !text.starts_with(prefix.as_str()) + { + text = format!("{prefix}{text}"); + } + + // `_build_provider_specific_fields`, unset fields left out. + let mut provider_fields = Map::new(); + let mut put = |key: &str, value: Option| { + if let Some(value) = value.filter(|value| !value.is_null()) { + provider_fields.insert(key.to_owned(), value); + } + }; + put("citations", citations.map(Value::Array)); + put("thinking_blocks", thinking_blocks.clone().map(Value::Array)); + put("context_management", body.context_management.clone()); + put("web_search_results", web_search_results.map(Value::Array)); + put("tool_results", tool_results.map(Value::Array)); + put("container", body.container.clone()); + put("compaction_blocks", compaction_blocks.map(Value::Array)); + let refusal = refusal_fields(body.stop_reason.as_deref(), body.stop_details.as_ref()); + provider_fields.extend(refusal.clone()); + + let thinking_blocks: Option> = thinking_blocks + .map(|blocks| serde_json::from_value(Value::Array(blocks))) + .transpose() + .map_err(|error| Error::InvalidResponse(ErrorDetail::invalid("thinking block", error)))?; + + let mut stop_reason = body.stop_reason.clone().unwrap_or_default(); + let json_indices: Vec = match shape.json_mode { + true => tool_calls + .iter() + .enumerate() + .filter(|(_, call)| call.function.name.as_deref() == Some(RESPONSE_FORMAT_TOOL_NAME)) + .map(|(index, _)| index) + .collect(), + false => Vec::new(), + }; + let schema = shape.response_format_schema.as_ref(); + let message = if !json_indices.is_empty() && json_indices.len() == tool_calls.len() { + stop_reason = "stop".into(); + let arguments: Vec = tool_calls + .iter() + .map(|call| call.function.arguments.clone()) + .collect(); + ChatCompletionsChoiceMessage { + role: "assistant".into(), + content: Some(json_tool_answer(&arguments, schema)), + provider_specific_fields: (!refusal.is_empty()).then_some(refusal), + ..ChatCompletionsChoiceMessage::default() + } + } else { + if !json_indices.is_empty() { + let arguments: Vec = json_indices + .iter() + .map(|index| tool_calls[*index].function.arguments.clone()) + .collect(); + text.push_str(&json_tool_answer(&arguments, schema)); + let mut position = 0; + tool_calls.retain(|_| { + let keep = !json_indices.contains(&position); + position += 1; + keep + }); + } + ChatCompletionsChoiceMessage { + role: "assistant".into(), + content: (!text.is_empty()).then_some(text), + tool_calls: (!tool_calls.is_empty()).then_some(tool_calls), + reasoning_content, + thinking_blocks, + provider_specific_fields: (!provider_fields.is_empty()).then_some(provider_fields), + } + }; + + Ok(ChatCompletionsResponse { + created: unix_now(), + model: body.model, + choices: vec![ChatCompletionsChoice { + index: 0, + message, + finish_reason: finish_reason_for(&stop_reason).to_string(), + }], + usage: usage_from_parts( + body.usage.input_tokens, + body.usage.output_tokens, + body.usage.cache_read_input_tokens.unwrap_or(0), + body.usage.cache_creation_input_tokens.unwrap_or(0), + ), + }) +} + impl BaseConfig for AnthropicConfig { fn secret_names(&self) -> Vec<&'static str> { use crate::anthropic::common_utils::{ @@ -96,10 +798,6 @@ impl BaseConfig for AnthropicConfig { ] } - fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)] { - SUPPORTED_PARAMS - } - fn get_complete_url( &self, api_base: Option<&str>, @@ -115,59 +813,27 @@ impl BaseConfig for AnthropicConfig { model: &str, messages: Vec, optional_params: Map, + context: &ChatRequestContext<'_>, ) -> Result { - Ok(ProviderChatRequestData { - body: anthropic_body(model, &build_conversation(&messages), optional_params), - stream_shape: StreamShape::default(), - }) + self.build_request(model, messages, optional_params, context) } fn transform_response( &self, _model: &str, response: ProviderChatResponseData, + shape: &StreamShape, ) -> Result { - let body: TextResponseProjection = - serde_json::from_value(response.body).map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid("messages response", error)) - })?; - // The route declines tool and thinking requests, so a non-text block - // means the response carries something this path never asked for. - // Decline rather than silently dropping it; the host falls back. - if body - .content - .iter() - .any(|block| matches!(block, TextResponseBlock::Other)) - { - return Err(Error::Unsupported("non-text response content block")); - } - let text: String = body - .content - .into_iter() - .map(|block| match block { - TextResponseBlock::Text { text } => text, - TextResponseBlock::Other => String::new(), - }) - .collect(); + let body: ResponseProjection = serde_json::from_value(response.body).map_err(|error| { + Error::InvalidResponse(ErrorDetail::invalid("messages response", error)) + })?; + anthropic_response(body, shape) + } - Ok(ChatCompletionsResponse { - created: unix_now(), - model: body.model, - choices: vec![ChatCompletionsChoice { - index: 0, - message: ChatCompletionsChoiceMessage { - role: "assistant".to_string(), - content: (!text.is_empty()).then_some(text), - }, - finish_reason: finish_reason_for(body.stop_reason.as_deref().unwrap_or("")) - .to_string(), - }], - usage: usage_from_parts( - body.usage.input_tokens, - body.usage.output_tokens, - body.usage.cache_read_input_tokens.unwrap_or(0), - body.usage.cache_creation_input_tokens.unwrap_or(0), - ), + /// `inlines_remote_media`: Anthropic fetches an `https` image itself, not an `http` one. + fn remote_media(&self, messages: &[ChatMessage]) -> Vec { + remote_urls(messages, &["user"], false, |url| { + url.starts_with("http://") && data_url(url).is_none() }) } @@ -214,107 +880,105 @@ impl BaseConfig for AnthropicConfig { DEFAULT_HEADERS } - /// An OAuth bearer is the whole credential: Python's `validate_environment` - /// authenticates with it and drops `x-api-key` rather than resolving one, so - /// the resolved key must not be applied over the top. Any other forwarded - /// `authorization` is unrelated to this header and does not defer, which is - /// also what Python does: it sends the deployment's `x-api-key` alongside. + /// Every request is translated; a param or message Python refuses is refused here too. fn unsupported_reason( &self, - messages: &[ChatMessage], - optional_params: &Map, + _messages: &[ChatMessage], + _optional_params: &Map, ) -> Option { - unsupported_param(self.supported_openai_param_mappings(), &[], optional_params) - .or_else(|| messages.iter().find_map(unsupported_message)) - // Anthropic rejects a request whose first turn is not a user turn. - // Python only repairs that under `litellm.modify_params`, which the - // core cannot observe, so decline instead of guessing. - .or_else(|| { - (!build_conversation(messages).opens_on_user_turn()) - .then_some(Unsupported("conversation does not open on a user turn")) - }) + None } } -fn text_block(text: &str) -> Value { - json!({"type": "text", "text": text}) -} - -fn anthropic_body( - model: &str, - conversation: &Conversation, - optional_params: Map, -) -> Value { - let messages: Vec = conversation - .turns - .iter() - .map(|turn| { - json!({ - "role": <&'static str>::from(turn.role), - "content": turn.texts.iter().map(|text| text_block(text)).collect::>(), - }) - }) - .collect(); - - let system: Vec = conversation.system.iter().map(|s| text_block(s)).collect(); - - let body = Map::from_iter( - [ - ("model".to_string(), json!(model)), - ("messages".to_string(), json!(messages)), - ] - .into_iter() - // Python builds `{"model", "messages", **optional_params}` with - // `system` already folded into optional_params, so a caller-supplied - // key of the same name wins here too. - .chain((!system.is_empty()).then(|| ("system".to_string(), json!(system)))) - .chain(optional_params), - ); - Value::Object(body) -} - #[cfg(test)] mod tests { - use crate::{ - Error, - anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, - base_llm::{ - auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, - }, - }; + use std::collections::HashMap; + use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; use rstest::rstest; use serde_json::{Map, Value, json}; + use super::*; + use crate::base_llm::model_info::ModelInfo; + + struct Map1(HashMap>); + + impl ModelInfo for Map1 { + fn entry(&self, model: &str) -> Option>> { + self.0.get(model).map(std::borrow::Cow::Borrowed) + } + fn declares(&self, key: &str) -> bool { + self.0.values().any(|entry| entry.contains_key(key)) + } + } + + fn cost_map() -> Map1 { + let entry = |value: Value| value.as_object().cloned().unwrap(); + Map1(HashMap::from([ + ( + "claude-sonnet-4-5".to_owned(), + entry(json!({"max_output_tokens": 64000, "supports_reasoning": true})), + ), + ( + "claude-opus-4-8".to_owned(), + entry( + json!({"max_output_tokens": 128000, "supports_adaptive_thinking": true, + "supports_speed": true, "supports_sampling_params": false, + "supports_native_structured_output": true}), + ), + ), + ])) + } + fn messages(value: Value) -> Vec { serde_json::from_value(value).expect("valid messages") } fn params(value: Value) -> Map { - match value { - Value::Object(map) => map, - other => panic!("params must be an object, got {other}"), - } + value.as_object().cloned().expect("params object") + } + + fn request( + model: &str, + msgs: Value, + opts: Value, + drop: bool, + ) -> Result { + let info = cost_map(); + let context = ChatRequestContext { + drop_params: Some(drop), + model_info: Some(&info), + }; + ANTHROPIC_CHAT_COMPLETIONS_CONFIG.transform_request( + model, + messages(msgs), + params(opts), + &context, + ) } fn transform(model: &str, msgs: Value, opts: Value) -> Value { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG - .transform_request(model, messages(msgs), params(opts)) + request(model, msgs, opts, false) .expect("request transforms") .body } fn transform_response(body: Value) -> Result { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG - .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) + transform_response_with(body, &StreamShape::default()) } - fn reason(msgs: Value, opts: Value) -> Option { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + fn transform_response_with( + body: Value, + shape: &StreamShape, + ) -> Result { + ANTHROPIC_CHAT_COMPLETIONS_CONFIG.transform_response( + "claude-sonnet-4-5", + ProviderChatResponseData { body }, + shape, + ) } - #[test] + #[rstest] fn builds_the_messages_body_python_builds() { let body = transform( "claude-sonnet-4-5", @@ -322,7 +986,7 @@ mod tests { {"role": "system", "content": "be terse"}, {"role": "user", "content": "hi"} ]), - json!({"max_tokens": 128, "temperature": 0.2}), + json!({"max_tokens": 128, "temperature": 0.2, "stop": "END", "user": "u-1"}), ); assert_eq!( body, @@ -331,354 +995,466 @@ mod tests { "messages": [ {"role": "user", "content": [{"type": "text", "text": "hi"}]} ], - "system": [{"type": "text", "text": "be terse"}], + "temperature": 0.2, + "stop_sequences": ["END"], "max_tokens": 128, - "temperature": 0.2 + "metadata": {"user_id": "u-1"}, + "system": [{"type": "text", "text": "be terse"}] }) ); } - #[test] - fn omits_system_when_no_system_message_is_present() { + #[rstest] + #[case::from_the_cost_map("claude-sonnet-4-5", 64000)] + #[case::unknown_model("claude-unknown", 4096)] + fn max_tokens_defaults_like_python(#[case] model: &str, #[case] expected: u64) { + let body = transform(model, json!([{"role": "user", "content": "hi"}]), json!({})); + assert_eq!(body["max_tokens"], json!(expected)); let body = transform( - "claude-sonnet-4-5", + model, json!([{"role": "user", "content": "hi"}]), - json!({"max_tokens": 16}), + json!({"max_completion_tokens": 7}), ); - assert!(body.get("system").is_none()); + assert_eq!(body["max_tokens"], json!(7)); } - #[test] - fn merges_consecutive_turns_and_wraps_every_text_in_a_block() { + #[rstest] + fn tools_tool_choice_and_history_round_trip() { let body = transform( "claude-sonnet-4-5", json!([ - {"role": "user", "content": "one"}, - {"role": "user", "content": [{"type": "text", "text": "two"}]}, - {"role": "assistant", "content": "ack"} - ]), - json!({"max_tokens": 16}), - ); - assert_eq!( - body["messages"], - json!([ - {"role": "user", "content": [ - {"type": "text", "text": "one"}, - {"type": "text", "text": "two"} + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_1", "type": "function", + "function": {"name": "get.weather", "arguments": "{\"city\":\"Paris\"}"}} ]}, - {"role": "assistant", "content": [{"type": "text", "text": "ack"}]} - ]) - ); - } - - #[test] - fn right_strips_a_trailing_assistant_prefill_like_python() { - let body = transform( - "claude-sonnet-4-5", - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "Argentina "} + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"} ]), - json!({"max_tokens": 16}), - ); - assert_eq!( - body["messages"][1]["content"][0]["text"], - json!("Argentina") - ); - } - - #[test] - fn passes_every_supported_param_through_untouched() { - let body = transform( - "claude-sonnet-4-5", - json!([{"role": "user", "content": "hi"}]), json!({ - "max_tokens": 64, - "temperature": 0.1, - "top_p": 0.9, - "stop_sequences": ["STOP"] + "max_tokens": 10, + "tools": [{"type": "function", "function": { + "name": "get.weather", "description": "d", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}} + }}], + "tool_choice": {"type": "function", "function": {"name": "get.weather"}}, + "parallel_tool_calls": false }), ); - assert_eq!(body["max_tokens"], json!(64)); - assert_eq!(body["temperature"], json!(0.1)); - assert_eq!(body["top_p"], json!(0.9)); - assert_eq!(body["stop_sequences"], json!(["STOP"])); - } - - #[test] - fn declines_top_k_because_python_gates_it_by_model_below_this_point() { - // `temperature` and `top_p` arrive already resolved, because - // `map_openai_params` applies `_apply_sampling_param` to them before the - // gate runs. `top_k` bypasses that and is gated inside `transform_request`, - // the function this route replaces, so forwarding it would send `top_k` to - // a model that removed sampling params and take a 400 after the call, where - // Python drops it and succeeds. assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"top_k": 40}) - ), - Some(Unsupported("unrecognized request parameter")) + body["tools"], + json!([{"name": "get_weather", "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, "type": "custom", "description": "d"}]) ); - } - - #[test] - fn declines_streaming_before_anything_else() { assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": true, "max_tokens": 16}) - ), - Some(Unsupported("streaming")) + body["tool_choice"], + json!({"type": "tool", "name": "get_weather", "disable_parallel_tool_use": true}) ); - } - - #[test] - fn accepts_an_explicit_stream_false() { assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": false, "max_tokens": 16}) - ), - None + body["messages"][1], + json!({"role": "assistant", "content": [ + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Paris"}} + ]}) + ); + assert_eq!( + body["messages"][2], + json!({"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"} + ]}) ); } #[rstest] - #[case::tools(json!({"tools": []}))] - #[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))] - #[case::thinking(json!({"thinking": {"type": "enabled"}}))] - #[case::system(json!({"system": "injected"}))] - #[case::metadata(json!({"metadata": {"user_id": "u1"}}))] - #[case::output_config(json!({"output_config": {"effort": "high"}}))] - fn declines_any_param_outside_the_allowlist(#[case] param: Value) { + fn tool_history_without_tools_gets_the_dummy_tool() { + let body = transform( + "claude-sonnet-4-5", + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "tool_calls": [ + {"id": "c", "type": "function", "function": {"name": "f", "arguments": ""}} + ]}, + {"role": "tool", "tool_call_id": "c", "content": "ok"} + ]), + json!({"max_tokens": 5}), + ); + assert_eq!(body["tools"][0]["name"], "dummy_tool"); + assert_eq!(body["tools"][0]["description"], "This is a dummy tool call"); + } + + #[rstest] + fn response_format_becomes_a_forced_json_tool() { + let data = request( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"max_tokens": 5, "response_format": {"type": "json_schema", "json_schema": { + "name": "x", "schema": {"type": "object", "properties": {"a": {"type": "string"}}} + }}}), + false, + ) + .unwrap(); assert_eq!( - reason(json!([{"role": "user", "content": "hi"}]), param.clone()), - Some(Unsupported("unrecognized request parameter")), - "expected {param} to decline" + data.body["tool_choice"], + json!({"name": "json_tool_call", "type": "tool"}) + ); + assert_eq!(data.body["tools"][0]["name"], "json_tool_call"); + assert!(data.body.get("json_mode").is_none()); + assert!(data.stream_shape.json_mode); + assert!(data.stream_shape.response_format_schema.is_some()); + } + + #[rstest] + fn native_structured_output_needs_the_beta() { + let data = request( + "claude-opus-4-8", + json!([{"role": "user", "content": "hi"}]), + json!({"max_tokens": 5, "response_format": {"type": "json_schema", "json_schema": { + "name": "x", "schema": {"type": "object", "properties": {"a": {"type": "string"}}} + }}}), + false, + ) + .unwrap(); + assert!(data.body.get("output_format").is_some()); + assert_eq!( + data.headers, + vec![( + "anthropic-beta".to_owned(), + "structured-outputs-2025-11-13".to_owned() + )] ); } - #[test] - fn declines_tool_calls_tool_results_and_multimodal_content() { + #[rstest] + fn unsupported_params_are_refused_or_dropped() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let error = request("claude-sonnet-4-5", msgs.clone(), json!({"seed": 1}), false) + .map(|_| ()) + .unwrap_err(); + assert!(matches!( + error, + Error::InvalidRequest(ErrorDetail::UnsupportedParams { ref params, .. }) if params == &["seed"] + )); + let body = request("claude-sonnet-4-5", msgs.clone(), json!({"seed": 1}), true) + .unwrap() + .body; + assert!(body.get("seed").is_none()); + let body = request( + "claude-sonnet-4-5", + msgs.clone(), + json!({"seed": 1, "drop_params": true}), + false, + ) + .unwrap() + .body; + assert!(body.get("seed").is_none()); + let body = request( + "claude-sonnet-4-5", + msgs.clone(), + json!({"seed": 1, "allowed_openai_params": ["seed"]}), + false, + ) + .unwrap() + .body; + assert_eq!(body["seed"], 1); + let body = request( + "claude-sonnet-4-5", + msgs, + json!({"temperature": 0.5, "additional_drop_params": ["temperature"]}), + false, + ) + .unwrap() + .body; + assert!(body.get("temperature").is_none()); + } + + #[rstest] + fn n_above_one_is_refused() { + let error = request( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"n": 2}), + false, + ) + .map(|_| ()) + .unwrap_err(); + assert!(matches!( + error, + Error::InvalidRequest(ErrorDetail::UnsupportedParams { .. }) + )); + } + + #[rstest] + fn sampling_params_follow_the_model() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let error = request("claude-opus-4-8", msgs.clone(), json!({"top_k": 3}), false) + .map(|_| ()) + .unwrap_err(); assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "c1", "type": "function", - "function": {"name": "f", "arguments": "{}"}} - ]} - ]), - json!({}) - ), - Some(Unsupported("unrecognized message field")) + error.to_string(), + "invalid request: claude-opus-4-8 does not support top_k=3. To drop unsupported params, set `litellm.drop_params = True`." ); + let body = request("claude-opus-4-8", msgs.clone(), json!({"top_k": 3}), true) + .unwrap() + .body; + assert!(body.get("top_k").is_none()); + let body = transform("claude-sonnet-4-5", msgs, json!({"top_k": 3})); + assert_eq!(body["top_k"], 3); + } + + #[rstest] + fn speed_is_checked_against_the_exact_entry() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let data = request( + "claude-opus-4-8", + msgs.clone(), + json!({"speed": "fast"}), + false, + ) + .unwrap(); + assert_eq!(data.body["speed"], "fast"); + assert_eq!(data.stream_shape.speed.as_deref(), Some("fast")); + assert_eq!(data.headers[0].1, "fast-mode-2026-02-01"); + let error = request("claude-sonnet-4-5", msgs, json!({"speed": "fast"}), false) + .map(|_| ()) + .unwrap_err(); assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "tool", "tool_call_id": "c1", "content": "ok"} - ]), - json!({}) - ), - Some(Unsupported("unrecognized message field")) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": [ - {"type": "image_url", "image_url": {"url": "https://x/y.png"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": [ - {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) + error.to_string(), + "invalid request: claude-sonnet-4-5 does not support speed='fast'. To drop unsupported params, set `litellm.drop_params = True`." ); } - #[test] - fn declines_a_message_whose_content_list_is_empty() { - // An empty list passes every per-part check, so without this it would reach - // the provider as an empty `content` array and fail after the call rather - // than declining to Python before it. - assert_eq!( - reason(json!([{"role": "user", "content": []}]), json!({})), - Some(Unsupported("message without content")) + #[rstest] + fn reasoning_effort_maps_to_thinking() { + let body = transform( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"reasoning_effort": "low"}), ); assert_eq!( - reason( - json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]), - json!({}) - ), - None + body["thinking"], + json!({"type": "enabled", "budget_tokens": 1024}) + ); + assert_eq!(body["max_tokens"], json!(1024 + 4096)); + } + + #[rstest] + fn output_config_effort_is_validated() { + let error = request( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"output_config": {"effort": "max"}}), + false, + ) + .map(|_| ()) + .unwrap_err(); + assert_eq!( + error.to_string(), + "invalid request: effort='max' is not supported by this model. Got model: claude-sonnet-4-5" + ); + let error = request( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"output_config": {"effort": "huge"}}), + false, + ) + .map(|_| ()) + .unwrap_err(); + assert!(error.to_string().contains("Invalid effort value: 'huge'")); + } + + #[rstest] + fn metadata_keeps_only_a_user_id() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let body = transform( + "claude-sonnet-4-5", + msgs.clone(), + json!({"metadata": {"user_id": "abc", "tags": ["x"]}}), + ); + assert_eq!(body["metadata"], json!({"user_id": "abc"})); + let body = transform("claude-sonnet-4-5", msgs, json!({"user": "a@b.com"})); + assert!(body.get("metadata").is_none()); + } + + #[rstest] + fn without_a_known_drop_policy_refusals_go_back_to_python() { + let context = ChatRequestContext::default(); + let error = ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .transform_request( + "claude-sonnet-4-5", + messages(json!([{"role": "user", "content": "hi"}])), + params(json!({"max_tokens": 3})), + &context, + ) + .map(|_| ()) + .unwrap_err(); + assert!(matches!(error, Error::Unsupported(_))); + } + + #[rstest] + fn only_http_images_are_downloaded() { + let urls = ANTHROPIC_CHAT_COMPLETIONS_CONFIG.remote_media(&messages(json!([ + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "http://x/a.png"}}, + {"type": "image_url", "image_url": {"url": "https://x/b.png"}} + ]} + ]))); + assert_eq!(urls, vec!["http://x/a.png"]); + } + + #[rstest] + fn a_data_url_image_becomes_a_base64_block() { + let body = transform( + "claude-sonnet-4-5", + json!([{"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} + ]}]), + json!({"max_tokens": 5}), + ); + assert_eq!( + body["messages"][0]["content"][1], + json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}}) ); } - #[test] - fn declines_a_conversation_that_does_not_open_on_a_user_turn() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "assistant", "content": "prefill"} - ]), - json!({}) - ), - Some(Unsupported("conversation does not open on a user turn")) - ); - } - - #[test] - fn accepts_a_plain_text_conversation() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "hello"}, - {"role": "user", "content": [{"type": "text", "text": "again"}]} - ]), - json!({"max_tokens": 16, "temperature": 0.5}) - ), - None - ); - } - - #[test] + #[rstest] fn normalizes_a_text_response_into_openai_shape() { let response = transform_response(json!({ "id": "msg_123", - "type": "message", - "role": "assistant", "model": "claude-sonnet-4-5-20260101", "content": [{"type": "text", "text": "hello"}, {"type": "text", "text": " there"}], "stop_reason": "end_turn", - "stop_sequence": null, "usage": {"input_tokens": 11, "output_tokens": 4} })) .expect("response transforms"); - assert_eq!(response.model, "claude-sonnet-4-5-20260101"); - assert_eq!(response.choices.len(), 1); - assert_eq!(response.choices[0].index, 0); - assert_eq!(response.choices[0].message.role, "assistant"); assert_eq!( response.choices[0].message.content.as_deref(), Some("hello there") ); assert_eq!(response.choices[0].finish_reason, "stop"); - assert_eq!(response.usage.prompt_tokens, 11); - assert_eq!(response.usage.completion_tokens, 4); assert_eq!(response.usage.total_tokens, 15); + let value = serde_json::to_value(response).unwrap(); + assert!(value.get("id").is_none()); + assert!(value["choices"][0]["message"].get("tool_calls").is_none()); } - #[test] + #[rstest] fn folds_cache_tokens_into_prompt_tokens_like_python() { let response = transform_response(json!({ "model": "claude-sonnet-4-5", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", - "usage": { - "input_tokens": 10, - "output_tokens": 2, - "cache_read_input_tokens": 5, - "cache_creation_input_tokens": 3 - } + "usage": {"input_tokens": 10, "output_tokens": 2, + "cache_read_input_tokens": 5, "cache_creation_input_tokens": 3} })) .expect("response transforms"); assert_eq!(response.usage.prompt_tokens, 18); - assert_eq!(response.usage.total_tokens, 20); assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5); assert_eq!( response.usage.prompt_tokens_details.cache_creation_tokens, 3 ); - assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10); } - #[test] - fn maps_max_tokens_stop_reason_to_length() { + #[rstest] + #[case::refusal("refusal", "content_filter")] + #[case::max_tokens("max_tokens", "length")] + #[case::tool_use("tool_use", "tool_calls")] + fn maps_stop_reasons(#[case] stop_reason: &str, #[case] expected: &str) { let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "max_tokens", + "model": "m", "content": [], "stop_reason": stop_reason, "usage": {"input_tokens": 1, "output_tokens": 1} })) - .expect("response transforms"); - assert_eq!(response.choices[0].finish_reason, "length"); - } - - #[test] - fn a_refusal_returns_the_completion_python_returns() { - // `refusal` is a stop_reason, not a content block type, so the content is - // ordinary text and this normalizes rather than declining. Python maps it - // to content_filter in _FINISH_REASON_MAP and returns the completion. - let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "I can't help with that."}], - "stop_reason": "refusal", - "usage": {"input_tokens": 9, "output_tokens": 6} - })) - .expect("a refusal still transforms"); - assert_eq!(response.choices[0].finish_reason, "content_filter"); - assert_eq!( - response.choices[0].message.content.as_deref(), - Some("I can't help with that.") - ); - } - - #[test] - fn reports_no_content_rather_than_an_empty_string() { - let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [], - "stop_reason": "end_turn", - "usage": {"input_tokens": 1, "output_tokens": 0} - })) - .expect("response transforms"); + .unwrap(); + assert_eq!(response.choices[0].finish_reason, expected); assert_eq!(response.choices[0].message.content, None); } - #[test] - fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() { - let response = transform_response(json!({ - "id": "msg_should_not_leak", - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "usage": {"input_tokens": 1, "output_tokens": 1} - })) - .expect("response transforms"); - let value = serde_json::to_value(response).expect("serializable"); - assert!( - value.get("id").is_none(), - "the rust response must not carry an id, got {value}" + #[rstest] + fn tool_use_and_thinking_come_back_as_openai_fields() { + let shape = StreamShape { + tool_name_reverse_map: HashMap::from([("get_weather".into(), "get.weather".into())]), + ..StreamShape::default() + }; + let response = transform_response_with( + json!({ + "model": "m", + "content": [ + {"type": "thinking", "thinking": "hmm", "signature": "sig"}, + {"type": "text", "text": "checking"}, + {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}} + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 1, "output_tokens": 1} + }), + &shape, + ) + .unwrap(); + let message = serde_json::to_value(&response.choices[0].message).unwrap(); + assert_eq!( + message["tool_calls"], + json!([{"id": "toolu_1", "type": "function", "function": {"name": "get.weather", "arguments": "{\"city\": \"Paris\"}"}, "index": 2}]) + ); + assert_eq!(message["reasoning_content"], "hmm"); + assert_eq!( + message["thinking_blocks"], + json!([{"type": "thinking", "thinking": "hmm", "signature": "sig"}]) + ); + assert_eq!(message["content"], "checking"); + assert_eq!(response.choices[0].finish_reason, "tool_calls"); + } + + #[rstest] + fn a_json_tool_answer_becomes_the_content() { + let shape = StreamShape { + json_mode: true, + ..StreamShape::default() + }; + let response = transform_response_with( + json!({ + "model": "m", + "content": [{"type": "tool_use", "id": "t", "name": "json_tool_call", "input": {"a": "b"}}], + "stop_reason": "tool_use", + "usage": {"input_tokens": 1, "output_tokens": 1} + }), + &shape, + ) + .unwrap(); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("{\"a\": \"b\"}") + ); + assert_eq!(response.choices[0].message.tool_calls, None); + assert_eq!(response.choices[0].finish_reason, "stop"); + } + + #[rstest] + fn a_prefix_prompt_is_put_back() { + let data = request( + "claude-sonnet-4-5", + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "Argentina", "prefix": true} + ]), + json!({"max_tokens": 5}), + false, + ) + .unwrap(); + assert_eq!( + data.stream_shape.prefix_prompt.as_deref(), + Some("Argentina") + ); + let response = transform_response_with( + json!({"model": "m", "content": [{"type": "text", "text": " is"}], + "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 1}}), + &data.stream_shape, + ) + .unwrap(); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("Argentina is") ); } - #[test] - fn declines_a_response_carrying_a_non_text_block() { - let err = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "tool_use", "id": "t1", "name": "f", "input": {}}], - "stop_reason": "tool_use", - "usage": {"input_tokens": 1, "output_tokens": 1} - })) - .expect_err("non-text block"); - assert_eq!(err, Error::Unsupported("non-text response content block")); - } - - #[rstest::rstest] + #[rstest] #[case::not_an_object(json!("nope"))] #[case::missing_content(json!({"model": "test-model", "usage": {"input_tokens": 1, "output_tokens": 1}}))] #[case::missing_usage(json!({"model": "test-model", "content": []}))] @@ -697,7 +1473,7 @@ mod tests { ); } - #[test] + #[rstest] fn resolves_the_messages_url_and_x_api_key_auth() { let config = &ANTHROPIC_CHAT_COMPLETIONS_CONFIG; assert_eq!( @@ -730,4 +1506,14 @@ mod tests { ] ); } + + #[rstest] + fn tool_names_are_sanitized_without_collisions() { + let (forward, reverse) = + tool_name_maps(&["a.b".to_owned(), "a_b".to_owned(), "c d".to_owned()]); + assert_eq!(forward["a.b"], "a_b_2"); + assert_eq!(forward["c d"], "c_d"); + assert!(!forward.contains_key("a_b")); + assert_eq!(reverse["a_b_2"], "a.b"); + } } diff --git a/litellm-rust/crates/llms/src/azure/chat/transformation.rs b/litellm-rust/crates/llms/src/azure/chat/transformation.rs index 7c6c1056f10..38df96b20cf 100644 --- a/litellm-rust/crates/llms/src/azure/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/azure/chat/transformation.rs @@ -8,56 +8,15 @@ use serde_json::{Map, Value}; use crate::{ Error, ErrorDetail, - base_llm::model_info::{self, ModelInfo}, + base_llm::{ + chat::params::{ + OPENAI_PARAMS, UNVALIDATED_PARAMS, ordered, request_drop_params, string_list, truthy, + }, + model_info::{self, ModelInfo}, + }, openai::chat::transformation::{is_litellm_owned, take_extra_body}, }; -/// The keys of `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` a request carries as OpenAI params, in -/// that table's order. Every other key goes to Azure through `extra_body`. -const OPENAI_PARAMS: &[&str] = &[ - "functions", - "function_call", - "temperature", - "top_p", - "n", - "stream", - "stream_options", - "stop", - "max_tokens", - "max_completion_tokens", - "modalities", - "prediction", - "audio", - "presence_penalty", - "frequency_penalty", - "logit_bias", - "user", - "response_format", - "seed", - "tools", - "tool_choice", - "max_retries", - "logprobs", - "top_logprobs", - "extra_headers", - "parallel_tool_calls", - "reasoning_effort", - "verbosity", - "thinking", - "web_search_options", - "include_server_side_tool_invocations", - "service_tier", - "safety_identifier", - "prompt_cache_key", - "prompt_cache_retention", - "store", - "metadata", - "context_management", -]; - -/// `PROVIDER_UNVALIDATED_PARAMS`: never refused, only left out when unsupported. -const UNVALIDATED_PARAMS: &[&str] = &["user", "stream_options", "stream", "max_retries"]; - /// `AzureOpenAIConfig.get_supported_openai_params`. const AZURE_GPT_PARAMS: &[&str] = &[ "temperature", @@ -410,6 +369,7 @@ impl AzureChatRequest<'_> { let model = self.deployment; if model.starts_with(RESPONSES_ROUTE) || model_info::field(self.info(), &format!("azure/{model}"), "mode") + .as_ref() .and_then(Value::as_str) == Some("responses") { @@ -830,7 +790,9 @@ fn openai_gpt_params(model: &str, info: Option<&dyn ModelInfo>) -> Vec<&'static .split_once(RESPONSES_ROUTE) .map_or(model, |(_, rest)| rest); if matches!( - model_info::field(info, catalog_name, "litellm_provider").and_then(Value::as_str), + model_info::field(info, catalog_name, "litellm_provider") + .as_ref() + .and_then(Value::as_str), Some("openai" | "text-completion-openai") ) { params.push("user"); @@ -913,14 +875,6 @@ fn azure_messages(messages: Value) -> Value { Value::Array(messages) } -/// The request's OpenAI params in `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` order, the order -/// Python maps them in. -fn ordered(params: &Map) -> impl Iterator { - OPENAI_PARAMS - .iter() - .filter_map(|name| params.get(*name).map(|value| (*name, value))) -} - /// `_map_openai_params`: every supported param still unmapped goes as sent. fn copy_supported( non_default: &mut Map, @@ -936,40 +890,6 @@ fn copy_supported( } } -/// Python's truthiness for a JSON value. -fn truthy(value: &Value) -> bool { - match value { - Value::Null => false, - Value::Bool(flag) => *flag, - Value::Number(number) => number.as_f64() != Some(0.0), - Value::String(text) => !text.is_empty(), - Value::Array(items) => !items.is_empty(), - Value::Object(object) => !object.is_empty(), - } -} - -/// `normalize_drop_params`: a boolean, or a config's `"true"`. -fn request_drop_params(value: Option) -> bool { - match value { - Some(Value::Bool(flag)) => flag, - Some(Value::String(text)) => text.trim().eq_ignore_ascii_case("true"), - _ => false, - } -} - -fn string_list(value: Option) -> Vec { - match value { - Some(Value::Array(items)) => items - .into_iter() - .filter_map(|item| match item { - Value::String(text) => Some(text), - _ => None, - }) - .collect(), - _ => Vec::new(), - } -} - #[cfg(test)] mod tests { use std::collections::HashMap; @@ -997,8 +917,8 @@ mod tests { } impl ModelInfo for Entries { - fn entry(&self, model: &str) -> Option<&Map> { - self.0.get(model) + fn entry(&self, model: &str) -> Option>> { + self.0.get(model).map(std::borrow::Cow::Borrowed) } fn declares(&self, key: &str) -> bool { diff --git a/litellm-rust/crates/llms/src/base_llm/chat/media.rs b/litellm-rust/crates/llms/src/base_llm/chat/media.rs new file mode 100644 index 00000000000..8f631b5bb89 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/chat/media.rs @@ -0,0 +1,209 @@ +//! Media a chat message points at. Python downloads an image or file URL a provider cannot +//! take by reference while it builds the request; here the host downloads them first, from +//! the list a config names, and hands the messages back with each one inlined as a data URL. + +use std::collections::HashMap; + +use litellm_llms_types::formats::chat_completions::{ChatMessage, ChatMessageContent}; +use serde_json::Value; + +/// Python's `MAX_IMAGE_URL_DOWNLOAD_SIZE_MB`. +pub const MAX_MEDIA_DOWNLOAD_BYTES: u64 = 50 * 1024 * 1024; + +/// A part's `image_url`, given as a string or as `{"url": ...}`. +pub fn image_url(part: &Value) -> Option<&str> { + let image_url = part.get("image_url")?; + image_url + .as_str() + .or_else(|| image_url.get("url").and_then(Value::as_str)) +} + +/// `data:;base64,` split into its media type and payload, the way Python's +/// `split("data:")[1].split(";base64,")` reads it. +pub fn data_url(url: &str) -> Option<(&str, &str)> { + let (_, rest) = url.split_once("data:")?; + let (media_type, data) = rest.split_once(";base64,")?; + (!data.contains(";base64,")).then_some((media_type, data)) +} + +fn parts(message: &ChatMessage) -> &[Value] { + match &message.content { + Some(ChatMessageContent::Parts(parts)) => parts, + _ => &[], + } +} + +/// The URLs of `image_url` parts (and, with `files`, of `file.file_id` / `file.file_data`) +/// in the messages `roles` names that `remote` says must be downloaded, in order, each once. +pub fn remote_urls( + messages: &[ChatMessage], + roles: &[&str], + files: bool, + remote: impl Fn(&str) -> bool, +) -> Vec { + let mut urls: Vec = Vec::new(); + let mut add = |url: &str| { + if remote(url) && !urls.iter().any(|seen| seen == url) { + urls.push(url.to_owned()); + } + }; + for message in messages + .iter() + .filter(|message| roles.contains(&message.role.as_str())) + { + for part in parts(message) { + match part.get("type").and_then(Value::as_str) { + Some("image_url") => { + if let Some(url) = image_url(part) { + add(url); + } + } + Some("file") if files => { + let file = part.get("file"); + let source = ["file_id", "file_data"].into_iter().find_map(|key| { + file.and_then(|file| file.get(key)).and_then(Value::as_str) + }); + if let Some(url) = source { + add(url); + } + } + _ => {} + } + } + } + urls +} + +/// Every reference to a downloaded URL replaced by its data URL. +pub fn inline_media(messages: &mut [ChatMessage], downloaded: &HashMap) { + if downloaded.is_empty() { + return; + } + for message in messages { + let Some(ChatMessageContent::Parts(parts)) = &mut message.content else { + continue; + }; + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("image_url") => match part.get_mut("image_url") { + Some(Value::String(url)) => { + if let Some(data) = downloaded.get(url.as_str()) { + *url = data.clone(); + } + } + Some(Value::Object(image)) => { + if let Some(Value::String(url)) = image.get_mut("url") + && let Some(data) = downloaded.get(url.as_str()) + { + *url = data.clone(); + } + } + _ => {} + }, + Some("file") => { + if let Some(Value::Object(file)) = part.get_mut("file") { + for key in ["file_id", "file_data"] { + if let Some(Value::String(url)) = file.get_mut(key) + && let Some(data) = downloaded.get(url.as_str()) + { + *url = data.clone(); + } + } + } + } + _ => {} + } + } + } +} + +/// The media type a download is inlined with: the response's own, or one guessed from the +/// URL's extension the way Python's `infer_content_type_from_url_and_content` falls back. +pub fn downloaded_media_type(url: &str, content_type: &str) -> String { + let declared = content_type + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + if !declared.is_empty() && declared != "application/octet-stream" && declared.contains('/') { + return declared; + } + let path = url + .split(['?', '#']) + .next() + .unwrap_or(url) + .to_ascii_lowercase(); + let extension = path.rsplit('.').next().unwrap_or_default(); + match extension { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "webp" => "image/webp", + "pdf" => "application/pdf", + "txt" => "text/plain", + "csv" => "text/csv", + "html" | "htm" => "text/html", + "md" => "text/markdown", + "mp4" => "video/mp4", + _ => "image/jpeg", + } + .to_owned() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn messages(value: Value) -> Vec { + serde_json::from_value(value).unwrap() + } + + #[rstest] + fn remote_urls_are_listed_once_and_inlined() { + let mut list = messages(json!([ + {"role": "user", "content": [ + {"type": "image_url", "image_url": "http://a/x.png"}, + {"type": "image_url", "image_url": {"url": "http://a/x.png"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}, + {"type": "file", "file": {"file_id": "https://b/doc.pdf"}} + ]}, + {"role": "assistant", "content": [{"type": "image_url", "image_url": "http://c"}]} + ])); + let urls = remote_urls(&list, &["user", "tool"], true, |url| { + url.starts_with("http") + }); + assert_eq!(urls, vec!["http://a/x.png", "https://b/doc.pdf"]); + let downloaded = HashMap::from([( + "http://a/x.png".to_owned(), + "data:image/png;base64,Zm8=".to_owned(), + )]); + inline_media(&mut list, &downloaded); + let content = serde_json::to_value(&list[0].content).unwrap(); + assert_eq!(content[0]["image_url"], "data:image/png;base64,Zm8="); + assert_eq!(content[1]["image_url"]["url"], "data:image/png;base64,Zm8="); + assert_eq!(content[3]["file"]["file_id"], "https://b/doc.pdf"); + } + + #[rstest] + #[case("data:image/png;base64,AAAA", Some(("image/png", "AAAA")))] + #[case("https://x", None)] + fn data_urls_split(#[case] url: &str, #[case] expected: Option<(&str, &str)>) { + assert_eq!(data_url(url), expected); + } + + #[rstest] + #[case("https://x/a.PNG?x=1", "", "image/png")] + #[case("https://x/a", "image/webp; charset=x", "image/webp")] + #[case("https://x/a.pdf", "application/octet-stream", "application/pdf")] + fn media_types_fall_back_to_the_extension( + #[case] url: &str, + #[case] content_type: &str, + #[case] expected: &str, + ) { + assert_eq!(downloaded_media_type(url, content_type), expected); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/chat/mod.rs b/litellm-rust/crates/llms/src/base_llm/chat/mod.rs index fa7df180f50..4c43dde0757 100644 --- a/litellm-rust/crates/llms/src/base_llm/chat/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/chat/mod.rs @@ -1,2 +1,5 @@ +pub mod media; +pub mod params; +pub mod response_format; pub mod streaming; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/base_llm/chat/params.rs b/litellm-rust/crates/llms/src/base_llm/chat/params.rs new file mode 100644 index 00000000000..9ffacff1611 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/chat/params.rs @@ -0,0 +1,434 @@ +//! Python's `get_optional_params` for the providers chat translates: the request's keys are +//! sorted into OpenAI params, keys LiteLLM consumes itself, and provider-specific keys that +//! pass through; an OpenAI param the provider does not take is dropped under `drop_params` +//! and refused otherwise. + +use serde_json::{Map, Value}; + +use crate::{ + Error, ErrorDetail, base_llm::model_info::ModelInfo, + openai::chat::transformation::is_litellm_owned, +}; + +/// The keys of `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` a request carries as OpenAI params, in +/// that table's order, which is the order Python maps them in. +pub const OPENAI_PARAMS: &[&str] = &[ + "functions", + "function_call", + "temperature", + "top_p", + "n", + "stream", + "stream_options", + "stop", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "audio", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "user", + "response_format", + "seed", + "tools", + "tool_choice", + "max_retries", + "logprobs", + "top_logprobs", + "extra_headers", + "parallel_tool_calls", + "reasoning_effort", + "verbosity", + "thinking", + "web_search_options", + "include_server_side_tool_invocations", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "metadata", + "context_management", +]; + +/// `PROVIDER_UNVALIDATED_PARAMS`: never refused, only left out when unsupported. +pub const UNVALIDATED_PARAMS: &[&str] = &["user", "stream_options", "stream", "max_retries"]; + +/// The tool Python adds when `response_format` asks for JSON and the model has no native +/// structured output. +pub const RESPONSE_FORMAT_TOOL_NAME: &str = "json_tool_call"; + +/// Python's `DEFAULT_MAX_TOKENS`, added to a thinking budget when the caller set no limit. +pub const DEFAULT_MAX_TOKENS: u64 = 4096; + +/// The request's OpenAI params in `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` order. +pub fn ordered(params: &Map) -> impl Iterator { + OPENAI_PARAMS + .iter() + .filter_map(|name| params.get(*name).map(|value| (*name, value))) +} + +/// Python's truthiness for a JSON value. +pub fn truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64() != Some(0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(object) => !object.is_empty(), + } +} + +/// `normalize_drop_params`: a boolean, or a config's `"true"`. +pub fn request_drop_params(value: Option) -> bool { + match value { + Some(Value::Bool(flag)) => flag, + Some(Value::String(text)) => text.trim().eq_ignore_ascii_case("true"), + _ => false, + } +} + +pub fn string_list(value: Option) -> Vec { + match value { + Some(Value::Array(items)) => items + .into_iter() + .filter_map(|item| match item { + Value::String(text) => Some(text), + _ => None, + }) + .collect(), + _ => Vec::new(), + } +} + +/// What the host tells a config about the request beyond its params. +#[derive(Clone, Copy, Default)] +pub struct ChatRequestContext<'a> { + /// `drop_params` from the proxy settings or the deployment. `None` when the host cannot + /// say, as when Python hosts the call: a request Python would drop or refuse a param of + /// then goes back to Python instead. + pub drop_params: Option, + /// The model cost map, which several param rules read. + pub model_info: Option<&'a dyn ModelInfo>, +} + +/// Whether a param the provider does not take is dropped, refused, or left to Python. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DropPolicy(Option); + +impl DropPolicy { + pub fn new(drop_params: Option) -> Self { + Self(drop_params) + } + + /// Whether unsupported params are dropped. + pub fn drops(self) -> bool { + self.0 == Some(true) + } + + /// Whether the host said what `drop_params` is. + pub fn known(self) -> bool { + self.0.is_some() + } + + /// A param Python drops under `drop_params` and refuses otherwise: `Ok` means drop it. + pub fn unsupported(self, refusal: impl FnOnce() -> Error) -> Result<(), Error> { + match self.0 { + Some(true) => Ok(()), + Some(false) => Err(refusal()), + None => Err(Error::Unsupported( + "a param Python drops or refuses by drop_params", + )), + } + } +} + +/// Python's `UnsupportedParamsError` for one param with its own message. +pub fn unsupported_value(message: impl Into) -> Error { + Error::InvalidRequest(ErrorDetail::Message(message.into())) +} + +/// A request's params as `get_optional_params` sorts them. +#[derive(Debug, Default)] +pub struct SortedParams { + /// The OpenAI params, refused or dropped ones removed. + pub openai: Map, + /// Every other key, sent to the provider as given. + pub provider: Map, + /// Keys the config consumes as call configuration. + pub config: Map, + /// `allowed_openai_params`: sent as given even when the provider does not list them. + pub allowed: Vec, + pub drop: DropPolicy, +} + +/// Sorts `params` and refuses or drops the OpenAI params `supported` leaves out. +pub fn sort_params( + params: Map, + config_params: &[&str], + context: &ChatRequestContext<'_>, + supported: impl FnOnce(&Map) -> Result, Error>, + provider: &'static str, + model: &str, +) -> Result { + let mut params = params; + let request_drop = request_drop_params(params.remove("drop_params")); + let drop = DropPolicy::new(match context.drop_params { + Some(configured) => Some(configured || request_drop), + None if request_drop => Some(true), + None => None, + }); + let allowed = string_list(params.remove("allowed_openai_params")); + let additional_drop = string_list(params.remove("additional_drop_params")); + let mut sorted = SortedParams { + allowed, + drop, + ..SortedParams::default() + }; + for (name, value) in params { + if config_params.contains(&name.as_str()) { + sorted.config.insert(name, value); + continue; + } + if value.is_null() || is_litellm_owned(&name) || additional_drop.contains(&name) { + continue; + } + match OPENAI_PARAMS.contains(&name.as_str()) { + true => sorted.openai.insert(name, value), + false => sorted.provider.insert(name, value), + }; + } + let supported = supported(&sorted.openai)?; + let unsupported: Vec = sorted + .openai + .iter() + .filter(|(name, _)| { + !supported.contains(&name.as_str()) && !sorted.allowed.iter().any(|a| a == *name) + }) + .map(|(name, _)| name.clone()) + .collect(); + let refused: Vec = unsupported + .iter() + .filter(|name| { + !UNVALIDATED_PARAMS.contains(&name.as_str()) + && !(name.as_str() == "n" + && sorted.openai.get(name.as_str()).and_then(Value::as_f64) == Some(1.0)) + }) + .cloned() + .collect(); + if !refused.is_empty() { + drop.unsupported(|| { + Error::InvalidRequest(ErrorDetail::UnsupportedParams { + provider, + params: refused.clone(), + model: model.to_string(), + }) + })?; + } + for name in unsupported { + sorted.openai.remove(&name); + } + Ok(sorted) +} + +/// A capability flag the cost map sets for the first of `candidates` it knows. +/// +/// Without a cost map Python's own lookups have nothing to go on either, so the answer is +/// `None`, except when the host could still fall back to Python, which does have one. +pub fn capability( + context: &ChatRequestContext<'_>, + candidates: &[String], + key: &str, +) -> Result, Error> { + let Some(info) = context.model_info else { + return match context.drop_params { + None => Err(Error::Unsupported( + "a request whose translation reads the model cost map", + )), + Some(_) => Ok(None), + }; + }; + Ok(candidates + .iter() + .filter_map(|candidate| info.entry(candidate)) + .find_map(|entry| entry.get(key).and_then(Value::as_bool))) +} + +/// A numeric field of the first of `candidates` the cost map knows. +pub fn model_field( + context: &ChatRequestContext<'_>, + candidates: &[String], + key: &str, +) -> Option { + let info = context.model_info?; + candidates + .iter() + .filter_map(|candidate| info.entry(candidate)) + .find_map(|entry| entry.get(key).filter(|value| !value.is_null()).cloned()) +} + +/// Whether the cost map has an entry for any of `candidates`. +pub fn model_known(context: &ChatRequestContext<'_>, candidates: &[String]) -> bool { + context + .model_info + .is_some_and(|info| candidates.iter().any(|c| info.entry(c).is_some())) +} + +/// `json.dumps(value)` with Python's default separators and ASCII escaping. +pub fn python_dumps(value: &Value) -> String { + litellm_python_compat::json::dumps(&litellm_python_compat::json::from_json(value.clone())) + .unwrap_or_else(|_| value.to_string()) +} + +/// `json.loads` of tool call arguments, as `parse_tool_call_arguments` reads them: empty +/// arguments are `{}`. +pub fn parse_tool_arguments(arguments: Option<&Value>) -> Result { + match arguments { + None | Some(Value::Null) => Ok(Value::Object(Map::new())), + Some(Value::String(text)) if text.trim().is_empty() => Ok(Value::Object(Map::new())), + Some(Value::String(text)) => serde_json::from_str(text).map_err(|error| { + Error::InvalidRequest(ErrorDetail::Message(format!( + "tool call arguments are not valid JSON: {error}" + ))) + }), + Some(other) => Ok(other.clone()), + } +} + +/// `is_thinking_enabled` over a params map. +pub fn is_thinking_enabled(params: &Map) -> bool { + let thinking = params.get("thinking"); + thinking == Some(&Value::Bool(true)) + || thinking + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("enabled") + || params + .get("reasoning_effort") + .is_some_and(|effort| !effort.is_null()) +} + +/// The JSON schema a `response_format` names: `response_schema`, or `json_schema.schema`. +pub fn response_format_schema(value: &Value) -> Option { + if let Some(schema) = value.get("response_schema") { + return Some(schema.clone()); + } + value + .get("json_schema") + .and_then(|json_schema| json_schema.get("schema")) + .cloned() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn object(value: Value) -> Map { + match value { + Value::Object(map) => map, + other => panic!("expected an object, got {other}"), + } + } + + fn sort(params: Value, drop_params: Option) -> Result { + sort_params( + object(params), + &["aws_region_name"], + &ChatRequestContext { + drop_params, + model_info: None, + }, + |_| Ok(vec!["temperature", "max_tokens"]), + "anthropic", + "claude", + ) + } + + #[rstest] + fn params_sort_into_openai_provider_config_and_owned() { + let sorted = sort( + json!({"temperature": 0.5, "top_k": 3, "aws_region_name": "us-east-1", + "metadata": {"x": 1}, "mock_response": "hi", "_litellm_x": 1, "user": "u", "seed": null}), + Some(false), + ) + .unwrap(); + assert_eq!(sorted.openai, object(json!({"temperature": 0.5}))); + assert_eq!(sorted.provider, object(json!({"top_k": 3}))); + assert_eq!( + sorted.config, + object(json!({"aws_region_name": "us-east-1"})) + ); + } + + #[rstest] + #[case::refused(Some(false), false)] + #[case::dropped(Some(true), true)] + #[case::request_drop(None, true)] + fn an_unsupported_param_is_refused_or_dropped( + #[case] drop_params: Option, + #[case] dropped: bool, + ) { + let mut body = json!({"temperature": 0.5, "seed": 3}); + if drop_params.is_none() { + body["drop_params"] = json!(true); + } + let result = sort(body, drop_params); + match dropped { + true => assert_eq!(result.unwrap().openai, object(json!({"temperature": 0.5}))), + false => assert!(matches!( + result, + Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams { ref params, .. })) + if params == &vec!["seed".to_string()] + )), + } + } + + #[rstest] + fn an_unknown_drop_policy_falls_back() { + assert!(matches!( + sort(json!({"seed": 3}), None), + Err(Error::Unsupported(_)) + )); + } + + #[rstest] + fn allowed_and_additional_drop_params_are_honoured() { + let sorted = sort( + json!({"seed": 3, "top_k": 2, "allowed_openai_params": ["seed"], "additional_drop_params": ["top_k"]}), + Some(false), + ) + .unwrap(); + assert_eq!(sorted.openai, object(json!({"seed": 3}))); + assert!(sorted.provider.is_empty()); + assert_eq!(sorted.allowed, vec!["seed".to_string()]); + } + + #[rstest] + #[case::n_of_one(json!({"n": 1}), true)] + #[case::n_of_two(json!({"n": 2}), false)] + #[case::user(json!({"user": "u"}), true)] + fn unvalidated_params_and_one_choice_are_never_refused( + #[case] body: Value, + #[case] accepted: bool, + ) { + let result = sort(body, Some(false)); + assert_eq!(result.is_ok(), accepted, "{result:?}"); + if let Ok(sorted) = result { + assert!(sorted.openai.is_empty()); + } + } + + #[rstest] + fn python_dumps_matches_json_dumps() { + assert_eq!( + python_dumps(&json!({"b": "é", "a": [1, 2.5]})), + r#"{"b": "\u00e9", "a": [1, 2.5]}"# + ); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/chat/response_format.rs b/litellm-rust/crates/llms/src/base_llm/chat/response_format.rs new file mode 100644 index 00000000000..a7f139c123a --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/chat/response_format.rs @@ -0,0 +1,445 @@ +//! `response_format` answers delivered through the `json_tool_call` tool, as Python's +//! `base_utils.py` reads them back: the schema's shape, the envelopes a model wraps the +//! answer in, and an answer split across several tool calls. + +use std::collections::BTreeSet; + +use serde_json::{Map, Value}; + +use crate::base_llm::chat::params::python_dumps; + +/// `ResponseFormatSchema`: what the requested schema declares, enough to tell an answer +/// from an envelope around it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResponseFormatSchema { + pub property_names: BTreeSet, + pub required_names: BTreeSet, + pub allows_undeclared_keys: bool, +} + +impl ResponseFormatSchema { + /// `response_format_schema`: `None` unless `value` is an object whose `properties`, if + /// present, is one too. + pub fn from_schema(value: &Value) -> Option { + let document = value.as_object()?; + let properties = match document.get("properties") { + None => Map::new(), + Some(Value::Object(properties)) => properties.clone(), + Some(_) => return None, + }; + let required_names = match document.get("required") { + Some(Value::Array(items)) if items.iter().all(Value::is_string) => items + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + _ => BTreeSet::new(), + }; + let additional = document.get("additionalProperties"); + let allows_undeclared_keys = additional == Some(&Value::Bool(true)) + || additional.is_some_and(Value::is_object) + || document + .get("patternProperties") + .and_then(Value::as_object) + .is_some_and(|patterns| !patterns.is_empty()); + Some(Self { + property_names: properties.keys().cloned().collect(), + required_names, + allows_undeclared_keys, + }) + } + + fn accepts_a_lone_undeclared_key(&self) -> bool { + self.allows_undeclared_keys && self.required_names.is_empty() + } +} + +const ENVELOPE_NAME_KEYS: [&str; 2] = ["name", "tool_name"]; + +/// `_json_object_or_encoded_object`. +fn object_or_encoded_object(value: &Value) -> Option> { + match value { + Value::String(text) => match serde_json::from_str(text) { + Ok(Value::Object(object)) => Some(object), + _ => None, + }, + Value::Object(object) => Some(object.clone()), + _ => None, + } +} + +struct EnvelopeLayer { + inner: Map, + names_the_tool: bool, +} + +fn envelope_layer( + payload: &Map, + schema: &ResponseFormatSchema, + tool_name: &str, +) -> Option { + let name_keys: BTreeSet<&str> = ENVELOPE_NAME_KEYS + .into_iter() + .filter(|key| payload.get(*key).and_then(Value::as_str) == Some(tool_name)) + .collect(); + let label_keys: BTreeSet<&str> = match (name_keys.is_empty(), schema.required_names.is_empty()) + { + (false, _) => name_keys.clone(), + (true, true) => BTreeSet::new(), + (true, false) => payload + .iter() + .filter(|(key, value)| { + value.is_string() + && !schema.property_names.contains(*key) + && !schema.required_names.contains(*key) + }) + .map(|(key, _)| key.as_str()) + .collect(), + }; + let wrapper_keys: Vec<&String> = payload + .keys() + .filter(|key| !label_keys.contains(key.as_str())) + .collect(); + let [wrapper] = wrapper_keys.as_slice() else { + return None; + }; + if schema.property_names.contains(*wrapper) { + return None; + } + let wrapped = &payload[wrapper.as_str()]; + let inner = match name_keys.is_empty() { + false => object_or_encoded_object(wrapped), + true => wrapped.as_object().cloned(), + }?; + Some(EnvelopeLayer { + inner, + names_the_tool: !name_keys.is_empty(), + }) +} + +fn holds_the_schema_object(layer: &EnvelopeLayer, schema: &ResponseFormatSchema) -> bool { + layer.names_the_tool + || (!schema.accepts_a_lone_undeclared_key() + && layer + .inner + .keys() + .any(|key| schema.property_names.contains(key))) +} + +fn schema_object_inside_envelope( + payload: &Map, + schema: &ResponseFormatSchema, + tool_name: &str, +) -> Option> { + let mut layer = envelope_layer(payload, schema, tool_name); + let mut first = true; + while let Some(current) = layer { + if !first && schema.property_names.is_empty() { + return None; + } + first = false; + if holds_the_schema_object(¤t, schema) { + return Some(current.inner); + } + layer = envelope_layer(¤t.inner, schema, tool_name); + } + None +} + +/// `response_format_envelope_contents`. +pub fn envelope_contents( + payload: &Value, + schema: &ResponseFormatSchema, + tool_name: &str, +) -> Option { + let object = payload.as_object()?; + if let Some(inner) = schema_object_inside_envelope(object, schema, tool_name) { + return Some(Value::Object(inner)); + } + if schema.property_names.contains("values") { + return None; + } + object + .get("values") + .filter(|values| !values.is_null()) + .cloned() +} + +/// `unwrap_response_format_envelope`. +pub fn unwrap_envelope(payload: Value, schema: &ResponseFormatSchema, tool_name: &str) -> Value { + envelope_contents(&payload, schema, tool_name).unwrap_or(payload) +} + +fn field_names( + blocks: &[Map], + key: &str, + schema: &ResponseFormatSchema, +) -> Option> { + let names: Vec = blocks + .iter() + .filter_map(|block| block[key].as_str()) + .filter(|name| schema.property_names.contains(*name)) + .map(str::to_owned) + .collect(); + let distinct: BTreeSet<&String> = names.iter().collect(); + (distinct.len() == blocks.len()).then_some(names) +} + +fn is_a_label_every_block_repeats(blocks: &[Map], key: &str) -> bool { + blocks[0][key].is_string() && blocks.iter().all(|block| block[key] == blocks[0][key]) +} + +fn value_key(blocks: &[Map], keys: &[&String]) -> Option { + if let [only] = keys { + return Some((*only).clone()); + } + let varying: Vec<&&String> = keys + .iter() + .filter(|key| !is_a_label_every_block_repeats(blocks, key)) + .collect(); + match varying.as_slice() { + [only] => Some((**only).clone()), + _ => None, + } +} + +fn split_answer_fields( + blocks: &[Map], + schema: &ResponseFormatSchema, +) -> Option> { + let keys: BTreeSet<&String> = blocks[0].keys().collect(); + if keys.iter().any(|key| schema.property_names.contains(*key)) + || blocks + .iter() + .any(|block| block.keys().collect::>() != keys) + { + return None; + } + // Python iterates a frozenset here; the outcome only depends on there being one match. + let named: Vec<(&String, Vec)> = keys + .iter() + .filter_map(|key| Some((*key, field_names(blocks, key, schema)?))) + .collect(); + let [(name_key, names)] = named.as_slice() else { + return None; + }; + let rest: Vec<&String> = keys.iter().copied().filter(|key| key != name_key).collect(); + let value_key = value_key(blocks, &rest)?; + Some( + names + .iter() + .cloned() + .zip(blocks.iter().map(|block| block[value_key.as_str()].clone())) + .collect(), + ) +} + +fn merged_partial_answers( + blocks: &[Map], + schema: &ResponseFormatSchema, +) -> Option> { + let keys: BTreeSet<&String> = blocks.iter().flat_map(Map::keys).collect(); + if !keys.iter().all(|key| schema.property_names.contains(*key)) + || keys.len() != blocks.iter().map(Map::len).sum::() + { + return None; + } + Some( + blocks + .iter() + .flat_map(|block| block.iter().map(|(k, v)| (k.clone(), v.clone()))) + .collect(), + ) +} + +/// `split_response_format_answer`: one answer spread over several tool calls. +pub fn split_answer( + encoded_blocks: &[String], + schema: &ResponseFormatSchema, +) -> Option> { + let blocks: Vec> = encoded_blocks + .iter() + .map(|encoded| object_or_encoded_object(&Value::String(encoded.clone()))) + .collect::>()?; + if blocks.len() < 2 { + return None; + } + split_answer_fields(&blocks, schema).or_else(|| merged_partial_answers(&blocks, schema)) +} + +/// Anthropic's `_convert_tool_response_to_message`: the message content the `json_tool_call` +/// calls' arguments become. +pub fn json_tool_answer(arguments: &[String], schema: Option<&ResponseFormatSchema>) -> String { + let first = arguments.first().cloned().unwrap_or_default(); + if let Some(split) = schema.and_then(|schema| split_answer(arguments, schema)) { + return python_dumps(&Value::Object(split)); + } + let Ok(payload) = serde_json::from_str::(&first) else { + return first; + }; + match schema { + Some(schema) => python_dumps(&unwrap_envelope( + payload, + schema, + crate::base_llm::chat::params::RESPONSE_FORMAT_TOOL_NAME, + )), + None => { + let values = payload + .as_object() + .and_then(|object| object.get("values")) + .filter(|values| !values.is_null()) + .cloned(); + python_dumps(values.as_ref().unwrap_or(&payload)) + } + } +} + +/// `unpack_defs`: every `$ref` into `$defs` / `definitions` replaced by a copy of its +/// target, nested definitions shadowing outer ones, a circular reference left as it is. +pub fn unpack_defs(schema: &mut Value, defs: &Map) { + fn scoped(node: &Map, outer: &Map) -> Map { + let mut defs = outer.clone(); + for key in ["$defs", "definitions"] { + if let Some(Value::Object(local)) = node.get(key) { + defs.extend(local.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + } + defs + } + fn walk(node: &mut Value, defs: &Map, chain: &BTreeSet) { + match node { + Value::Object(object) => { + if let Some(reference) = object.get("$ref").and_then(Value::as_str) { + let name = reference.rsplit('/').next().unwrap_or(reference).to_owned(); + if chain.contains(&name) { + return; + } + let Some(target) = defs.get(&name).cloned() else { + return; + }; + let child_defs = match &target { + Value::Object(target) => scoped(target, defs), + _ => defs.clone(), + }; + *node = target; + let mut chain = chain.clone(); + chain.insert(name); + walk(node, &child_defs, &chain); + return; + } + if object.contains_key("$ref") { + return; + } + let current = scoped(object, defs); + for value in object.values_mut() { + walk(value, ¤t, chain); + } + } + Value::Array(items) => { + for item in items { + walk(item, defs, chain); + } + } + _ => {} + } + } + let root = match schema { + Value::Object(object) => scoped(object, defs), + _ => defs.clone(), + }; + walk(schema, &root, &BTreeSet::new()); +} + +/// `unpack_legacy_defs`: refs into `definitions` or OpenAPI `components.schemas` inlined, +/// those blocks removed. `$defs` is left alone. +pub fn unpack_legacy_defs(schema: &mut Map) { + let has_components = schema + .get("components") + .and_then(|components| components.get("schemas")) + .is_some(); + if !schema.contains_key("definitions") && !has_components { + return; + } + let mut defs = match schema.remove("components") { + Some(Value::Object(mut components)) => match components.remove("schemas") { + Some(Value::Object(schemas)) => schemas, + _ => Map::new(), + }, + _ => Map::new(), + }; + if let Some(Value::Object(definitions)) = schema.remove("definitions") { + defs.extend(definitions); + } + let mut value = Value::Object(std::mem::take(schema)); + unpack_defs(&mut value, &defs); + if let Value::Object(object) = value { + *schema = object; + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn schema(value: Value) -> ResponseFormatSchema { + ResponseFormatSchema::from_schema(&value).unwrap() + } + + #[rstest] + #[case::bare(json!({"name": "a"}), json!({"name": "a"}))] + #[case::named_envelope(json!({"name": "json_tool_call", "arguments": {"name": "a"}}), json!({"name": "a"}))] + #[case::encoded_envelope(json!({"tool_name": "json_tool_call", "parameters": "{\"name\": \"a\"}"}), json!({"name": "a"}))] + #[case::wrapper(json!({"answer": {"name": "a"}}), json!({"name": "a"}))] + #[case::values(json!({"values": {"x": 1}}), json!({"x": 1}))] + fn envelopes_are_unwrapped(#[case] payload: Value, #[case] expected: Value) { + let schema = schema( + json!({"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}), + ); + assert_eq!( + unwrap_envelope(payload, &schema, "json_tool_call"), + expected + ); + } + + #[rstest] + fn an_answer_split_over_named_fields_is_joined() { + let schema = schema(json!({"properties": {"a": {}, "b": {}}})); + let blocks = vec![ + r#"{"field": "a", "value": 1}"#.to_owned(), + r#"{"field": "b", "value": 2}"#.to_owned(), + ]; + assert_eq!( + split_answer(&blocks, &schema), + Some(json!({"a": 1, "b": 2}).as_object().unwrap().clone()) + ); + let partial = vec![r#"{"a": 1}"#.to_owned(), r#"{"b": 2}"#.to_owned()]; + assert_eq!( + json_tool_answer(&partial, Some(&schema)), + r#"{"a": 1, "b": 2}"# + ); + } + + #[rstest] + fn an_undecodable_answer_is_returned_as_sent() { + assert_eq!(json_tool_answer(&["{bad".to_owned()], None), "{bad"); + assert_eq!( + json_tool_answer(&[r#"{"values": [1]}"#.to_owned()], None), + "[1]" + ); + } + + #[rstest] + fn refs_are_inlined_and_cycles_left_alone() { + let mut value = + json!({"properties": {"a": {"$ref": "#/$defs/A"}, "n": {"$ref": "#/$defs/Node"}}}); + let defs = json!({"A": {"type": "string"}, "Node": {"properties": {"next": {"$ref": "#/$defs/Node"}}}}); + unpack_defs(&mut value, defs.as_object().unwrap()); + assert_eq!( + value, + json!({"properties": {"a": {"type": "string"}, "n": {"properties": {"next": {"$ref": "#/$defs/Node"}}}}}) + ); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/chat/streaming.rs b/litellm-rust/crates/llms/src/base_llm/chat/streaming.rs index 2faa5699751..1c938b33a1e 100644 --- a/litellm-rust/crates/llms/src/base_llm/chat/streaming.rs +++ b/litellm-rust/crates/llms/src/base_llm/chat/streaming.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use crate::base_llm::chat::response_format::ResponseFormatSchema; + use bytes::Bytes; use futures_util::{StreamExt, stream::BoxStream}; use litellm_llms_types::formats::chat_completions::ChatCompletionChunk; @@ -35,6 +37,12 @@ pub struct StreamShape { pub json_mode: bool, pub speed: Option, pub tool_name_reverse_map: HashMap, + /// The schema of the `json_tool_call` tool a `response_format` added, which tells its + /// answer apart from an envelope around it. + pub response_format_schema: Option, + /// The text of a trailing assistant message sent with `prefix: true`, which Anthropic + /// continues and Python puts back in front of the answer. + pub prefix_prompt: Option, } /// A wire decoder paired with the iterator that turns its events into chat chunks. diff --git a/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs index e70001bf0be..77c89e0e85a 100644 --- a/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs @@ -5,7 +5,10 @@ use serde_json::{Map, Value}; use crate::{ Error, - base_llm::chat::streaming::{ChatStream, StreamShape}, + base_llm::chat::{ + params::ChatRequestContext, + streaming::{ChatStream, StreamShape}, + }, }; /// The provider-shaped request body a config produces. Named rather than a bare @@ -14,6 +17,8 @@ use crate::{ pub struct ProviderChatRequestData { pub body: Value, pub stream_shape: StreamShape, + /// Headers the request needs beyond the config's defaults, such as `anthropic-beta`. + pub headers: Vec<(String, String)>, } /// The raw provider response body handed back to a config for normalization. @@ -42,8 +47,11 @@ pub struct Unsupported(pub &'static str); pub trait BaseConfig: Sync { fn secret_names(&self) -> Vec<&'static str>; - /// Supported OpenAI parameter names paired with their provider names. - fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)]; + /// Supported OpenAI parameter names paired with their provider names, for a config + /// whose requests the host checks against an allowlist. + fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)] { + &[] + } fn get_complete_url( &self, @@ -58,14 +66,23 @@ pub trait BaseConfig: Sync { model: &str, messages: Vec, optional_params: Map, + context: &ChatRequestContext<'_>, ) -> Result; + /// `shape` is what the request decided about its answer, the same one a stream reads. fn transform_response( &self, model: &str, response: ProviderChatResponseData, + shape: &StreamShape, ) -> Result; + /// The media URLs the messages point at that Python downloads while building this + /// provider's request. The host fetches them and inlines each as a data URL first. + fn remote_media(&self, _messages: &[ChatMessage]) -> Vec { + Vec::new() + } + /// `None` means this config has no streaming path yet, so the host keeps the request. fn model_response_iterator(&self, _shape: StreamShape) -> Option { None diff --git a/litellm-rust/crates/llms/src/base_llm/model_info.rs b/litellm-rust/crates/llms/src/base_llm/model_info.rs index a2861d138c7..3260c270306 100644 --- a/litellm-rust/crates/llms/src/base_llm/model_info.rs +++ b/litellm-rust/crates/llms/src/base_llm/model_info.rs @@ -1,11 +1,14 @@ //! What the model cost map says about a model, for configs whose request rules depend on it //! the way Python's `supports_*` helpers read `litellm.model_cost`. +use std::borrow::Cow; + use serde_json::{Map, Value}; pub trait ModelInfo: Send + Sync { - /// The cost map entry `model` names, if there is one. - fn entry(&self, model: &str) -> Option<&Map>; + /// The cost map entry `model` names, if there is one. A map held in memory lends it; one + /// read from elsewhere, such as Python's `litellm.model_cost`, hands over a copy. + fn entry(&self, model: &str) -> Option>>; /// Whether any entry of the map sets `key`. fn declares(&self, key: &str) -> bool; @@ -14,21 +17,22 @@ pub trait ModelInfo: Send + Sync { /// Python's `_supports_factory`: the entry sets `key` to `true`. An unknown model supports /// nothing. pub fn supports(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> bool { - field(info, model, key).and_then(Value::as_bool) == Some(true) + field(info, model, key).and_then(|value| value.as_bool()) == Some(true) } /// Python's `is_explicitly_disabled_factory`: the entry sets `key` to `false`. An unknown /// model or an unset key is not disabled. pub fn explicitly_disabled(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> bool { - field(info, model, key).and_then(Value::as_bool) == Some(false) + field(info, model, key).and_then(|value| value.as_bool()) == Some(false) } /// The value the entry for `model` sets for `key`. -pub fn field<'a>(info: Option<&'a dyn ModelInfo>, model: &str, key: &str) -> Option<&'a Value> { +pub fn field(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> Option { info? .entry(model)? .get(key) .filter(|value| !value.is_null()) + .cloned() } /// Whether the map has an entry for `model`. diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_messages.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_messages.rs new file mode 100644 index 00000000000..67cbd088a7a --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_messages.rs @@ -0,0 +1,1343 @@ +//! Chat messages and tools as Converse takes them: Python's `_transform_system_message`, +//! `_neutralize_orphaned_tool_blocks`, `_bedrock_converse_messages_pt` and `_bedrock_tools_pt`. + +use std::collections::HashSet; + +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; + +use crate::{ + Error, ErrorDetail, + anthropic::chat::messages::{CONVERTED_SYSTEM_NOTE, native_redacted_thinking_data}, + base_llm::chat::{params::truthy, response_format::unpack_defs}, + bedrock::chat::converse_params::{BedrockModel, make_valid_bedrock_tool_name}, +}; + +const SUPPORTED_IMAGE_FORMATS: [&str; 4] = ["png", "jpeg", "gif", "webp"]; +const SUPPORTED_DOCUMENT_FORMATS: [&str; 9] = [ + "pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md", +]; +const SUPPORTED_VIDEO_FORMATS: [&str; 9] = [ + "mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp", +]; +const DOCUMENT_PLACEHOLDER_TEXT: &str = "."; +const TOOL_RESULT_IMAGE_PLACEHOLDER: &str = + "[Tool returned an image - see the following user message]"; +const TOOL_USE_ID_MAX_LEN: usize = 64; +const TOOL_USE_ID_HASH_LEN: usize = 8; +const HASH_SAMPLE_BYTES: usize = 64 * 1024; +const BAD_MESSAGE: &str = "Invalid Message "; + +fn bad_request(message: impl Into) -> Error { + Error::InvalidRequest(ErrorDetail::Message(message.into())) +} + +fn role(message: &Value) -> &str { + message + .get("role") + .and_then(Value::as_str) + .unwrap_or_default() +} + +fn is_system(message: &Value) -> bool { + role(message) == "system" +} + +/// `CachePointBlock` for a `cache_control`, or nothing for a model that refuses them. +pub fn cache_point(control: Option<&Value>, model: &BedrockModel<'_>) -> Option { + control.filter(|control| !control.is_null())?; + if !model.accepts_cache_points() { + return None; + } + Some(json!({"cachePoint": cache_point_block(control, model)})) +} + +fn cache_point_block(control: Option<&Value>, model: &BedrockModel<'_>) -> Value { + let mut block = Map::from_iter([("type".to_owned(), json!("default"))]); + if let Some(ttl) = control + .and_then(|control| control.get("ttl")) + .and_then(Value::as_str) + .filter(|ttl| matches!(*ttl, "5m" | "1h")) + && model.cache_ttl_supported() + { + block.insert("ttl".into(), json!(ttl)); + } + Value::Object(block) +} + +/// The leading run of system messages as Converse `system` blocks. +fn system_blocks(leading: &[Value], model: &BedrockModel<'_>) -> Vec { + let mut blocks = Vec::new(); + for message in leading { + match message.get("content") { + Some(Value::String(text)) if !text.is_empty() => { + blocks.push(json!({"text": text})); + blocks.extend(cache_point(message.get("cache_control"), model)); + } + Some(Value::Array(parts)) => { + for part in parts { + let text = part.get("text").and_then(Value::as_str).unwrap_or_default(); + if part.get("type").and_then(Value::as_str) == Some("text") && !text.is_empty() + { + blocks.push(json!({"text": text})); + blocks.extend(cache_point(part.get("cache_control"), model)); + } + } + } + _ => {} + } + } + blocks +} + +fn assistant_has_tool_calls(message: &Value) -> bool { + role(message) == "assistant" && message.get("tool_calls").is_some_and(truthy) +} + +/// `_opens_with_tool_result`. +fn opens_with_tool_result(message: &Value) -> bool { + match role(message) { + "tool" => true, + "user" => { + let first = match message.get("content") { + Some(Value::Array(parts)) => parts.first(), + _ => None, + }; + first + .and_then(|part| part.get("type")) + .and_then(Value::as_str) + == Some("tool_result") + } + _ => false, + } +} + +/// `_reordered_around_tool_results`. +fn reordered_around_tool_results(messages: &[Value], index: usize) -> Vec { + let message = &messages[index]; + let system_run_start = |end: usize| { + (0..end) + .rev() + .find(|j| !is_system(&messages[*j])) + .map_or(0, |j| j + 1) + }; + if opens_with_tool_result(message) { + if index + 1 < messages.len() && opens_with_tool_result(&messages[index + 1]) { + return vec![message.clone()]; + } + let tool_run_start = (0..=index) + .rev() + .find(|j| !opens_with_tool_result(&messages[*j])) + .map_or(0, |j| j + 1); + let run_start = system_run_start(tool_run_start); + let run = &messages[run_start..tool_run_start]; + if !run.is_empty() && run_start >= 1 && assistant_has_tool_calls(&messages[run_start - 1]) { + let mut kept = vec![message.clone()]; + kept.extend(run.iter().cloned()); + return kept; + } + return vec![message.clone()]; + } + if !is_system(message) { + return vec![message.clone()]; + } + let run_start = system_run_start(index); + let run_end = (index..messages.len()) + .find(|j| !is_system(&messages[*j])) + .unwrap_or(messages.len()); + if run_end < messages.len() + && opens_with_tool_result(&messages[run_end]) + && run_start > 0 + && assistant_has_tool_calls(&messages[run_start - 1]) + { + return Vec::new(); + } + vec![message.clone()] +} + +/// `_system_role_message_as_user`: a mid-conversation system message as a user turn. +fn system_as_user(message: &Value) -> Option { + let block = |text: &str, cache_control: Option<&Value>| { + let mut block = Map::from_iter([ + ("type".to_owned(), json!("text")), + ("text".to_owned(), json!(text)), + ]); + if let Some(cache_control) = cache_control.filter(|c| !c.is_null()) { + block.insert("cache_control".into(), cache_control.clone()); + } + Value::Object(block) + }; + let texts: Vec = match message.get("content") { + Some(Value::String(text)) if !text.is_empty() => { + vec![block(text, message.get("cache_control"))] + } + Some(Value::Array(parts)) => parts + .iter() + .filter(|part| part.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|part| { + let text = part + .get("text") + .and_then(Value::as_str) + .filter(|t| !t.is_empty())?; + Some(block(text, part.get("cache_control"))) + }) + .collect(), + _ => Vec::new(), + }; + if texts.is_empty() { + return None; + } + let mut content = vec![json!({"type": "text", "text": CONVERTED_SYSTEM_NOTE})]; + content.extend(texts); + Some(json!({"role": "user", "content": content})) +} + +/// `_transform_system_message`: the conversation after the leading system run, and that run +/// as `system` blocks. +pub fn transform_system_message( + messages: Vec, + model: &BedrockModel<'_>, +) -> (Vec, Vec) { + let leading = messages + .iter() + .position(|message| !is_system(message)) + .unwrap_or(messages.len()); + let system = system_blocks(&messages[..leading], model); + let remaining = &messages[leading..]; + let kept = (0..remaining.len()) + .flat_map(|index| reordered_around_tool_results(remaining, index)) + .filter_map(|message| match is_system(&message) { + true => system_as_user(&message), + false => Some(message), + }) + .collect(); + (kept, system) +} + +/// `convert_content_list_to_str`. +fn content_text(message: &Value) -> String { + match message.get("content") { + Some(Value::String(text)) => text.clone(), + Some(Value::Array(parts)) => parts + .iter() + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .collect(), + _ => String::new(), + } +} + +/// `_neutralize_orphaned_tool_blocks`: tool calls and results without `tools=` as text. +pub fn neutralize_orphaned_tool_blocks( + messages: Vec, + has_tools: bool, + guardrail_active: bool, +) -> Vec { + let orphaned = messages.iter().any(|message| { + assistant_has_tool_calls(message) || matches!(role(message), "tool" | "function") + }); + if has_tools || !orphaned { + return messages; + } + messages + .into_iter() + .map(|message| { + if assistant_has_tool_calls(&message) { + let base = content_text(&message); + let calls = message + .get("tool_calls") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|call| { + let function = call.get("function"); + let name = function + .and_then(|f| f.get("name")) + .and_then(Value::as_str) + .filter(|name| !name.is_empty()) + .unwrap_or("unknown_tool"); + let arguments = function + .and_then(|f| f.get("arguments")) + .and_then(Value::as_str) + .unwrap_or_default(); + let label = match call + .get("id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + { + Some(id) => format!("tool call {id}"), + None => "tool call".to_owned(), + }; + format!("[{label}: {name}({arguments})]") + }); + let text: Vec = std::iter::once(base) + .chain(calls) + .filter(|part| !part.is_empty()) + .collect(); + return json!({"role": "assistant", "content": text.join("\n")}); + } + if matches!(role(&message), "tool" | "function") { + let target = ["tool_call_id", "name"] + .iter() + .find_map(|key| { + message + .get(*key) + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + }) + .unwrap_or("unknown"); + let rendered = content_text(&message).trim().to_owned(); + let rendered = match rendered.is_empty() { + true => "".to_owned(), + false => rendered, + }; + let text = format!("[tool result for {target}: {rendered}]"); + return match guardrail_active { + true => { + json!({"role": "user", "content": [{"type": "guarded_text", "text": text}]}) + } + false => json!({"role": "user", "content": text}), + }; + } + message + }) + .collect() +} + +/// `_convert_consecutive_user_messages_to_guarded_text`. +pub fn guard_trailing_user_messages(mut messages: Vec) -> Vec { + for message in messages.iter_mut().rev() { + if role(message) != "user" { + break; + } + match message.get_mut("content") { + Some(Value::Array(parts)) => { + if parts + .iter() + .any(|part| part.get("type").and_then(Value::as_str) == Some("guarded_text")) + { + continue; + } + for part in parts.iter_mut() { + if part.get("type").and_then(Value::as_str) == Some("text") { + let text = part.get("text").cloned().unwrap_or(Value::Null); + *part = json!({"type": "guarded_text", "text": text}); + } + } + } + Some(Value::String(text)) => { + let text = text.clone(); + message["content"] = json!([{"type": "guarded_text", "text": text}]); + } + _ => {} + } + } + messages +} + +/// `_sanitize_bedrock_tool_use_id`. +pub fn sanitize_tool_use_id(id: &str) -> String { + let sanitized: String = id + .chars() + .map( + |c| match c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | ':' | '-') { + true => c, + false => '_', + }, + ) + .collect(); + if sanitized == id && sanitized.len() <= TOOL_USE_ID_MAX_LEN { + return sanitized; + } + let digest = hex(&Sha256::digest(id.as_bytes())); + let head: String = sanitized + .chars() + .take(TOOL_USE_ID_MAX_LEN - TOOL_USE_ID_HASH_LEN - 1) + .collect(); + format!("{head}_{}", &digest[..TOOL_USE_ID_HASH_LEN]) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// `mimetypes.guess_all_extensions` for the document types Converse takes. +fn document_format(mime_type: &str) -> Result { + let format = match mime_type { + "application/pdf" => "pdf", + "text/csv" => "csv", + "application/msword" => "doc", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => "docx", + "application/vnd.ms-excel" => "xls", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => "xlsx", + "text/html" => "html", + "text/plain" => "txt", + "text/markdown" | "text/x-markdown" => "md", + _ => { + return Err(bad_request(format!( + "No supported extensions for MIME type: {mime_type}. Supported formats: {}", + python_list(&SUPPORTED_DOCUMENT_FORMATS) + ))); + } + }; + Ok(format.to_owned()) +} + +fn python_list(items: &[&str]) -> String { + format!( + "[{}]", + items + .iter() + .map(|item| format!("'{item}'")) + .collect::>() + .join(", ") + ) +} + +fn is_document_mime(mime_type: &str) -> bool { + mime_type.starts_with("application") || mime_type.starts_with("text") +} + +/// `BedrockImageProcessor.validate_format`. +fn validate_format(mime_type: &str, image_format: &str) -> Result { + if is_document_mime(mime_type) { + return document_format(mime_type); + } + let supported: Vec<&str> = SUPPORTED_VIDEO_FORMATS + .iter() + .chain(SUPPORTED_IMAGE_FORMATS.iter()) + .copied() + .collect(); + if !supported.contains(&image_format) { + return Err(bad_request(format!( + "Unsupported image format: {image_format}. Supported formats: {}", + python_list(&supported) + ))); + } + Ok(image_format.to_owned()) +} + +/// The name Python derives from a document's content: a hash of its first 64 KB and length. +fn document_hash(data: &str) -> String { + let normalized: String = data.split_whitespace().collect(); + let bytes = normalized.as_bytes(); + let mut hasher = Sha256::new(); + hasher.update(&bytes[..bytes.len().min(HASH_SAMPLE_BYTES)]); + hasher.update(bytes.len().to_string().as_bytes()); + hex(&hasher.finalize())[..16].to_owned() +} + +/// `BedrockImageProcessor.process_image_sync` over a URL the host already inlined. +fn media_block(url: &str, format: Option<&str>) -> Result { + if !url.contains("base64") { + return Err(match url.contains("http://") || url.contains("https://") { + true => Error::Unsupported("a media URL the host did not download"), + false => bad_request( + "Unsupported image type. Expected either image url or base64 encoded string", + ), + }); + } + let (metadata, data) = url.split_once(',').unwrap_or((url, "")); + let (mut mime_type, mut image_format) = match metadata + .strip_prefix("data:") + .and_then(|rest| rest.split(";base64").next()) + { + Some(mime) => { + let mime = mime.split(';').next().unwrap_or(mime).to_owned(); + let format = mime.split('/').nth(1).unwrap_or_default().to_owned(); + (mime, format) + } + None => ("image/jpeg".to_owned(), "jpeg".to_owned()), + }; + if let Some(format) = format.filter(|format| !format.is_empty()) { + mime_type = format.to_owned(); + image_format = mime_type.split('/').nth(1).unwrap_or_default().to_owned(); + } + let image_format = validate_format(&mime_type, &image_format)?; + let source = json!({"bytes": data}); + if is_document_mime(&mime_type) { + let name = format!("DocumentPDFmessages_{}_{image_format}", document_hash(data)); + return Ok(json!({"document": {"source": source, "format": image_format, "name": name}})); + } + if SUPPORTED_VIDEO_FORMATS + .iter() + .any(|video| image_format.starts_with(video)) + { + return Ok(json!({"video": {"source": source, "format": image_format}})); + } + Ok(json!({"image": {"source": source, "format": image_format}})) +} + +fn image_url_block(part: &Value) -> Result { + let image_url = part.get("image_url"); + let (url, format) = match image_url { + Some(Value::Object(image)) => ( + image.get("url").and_then(Value::as_str).unwrap_or_default(), + image.get("format").and_then(Value::as_str), + ), + Some(Value::String(url)) => (url.as_str(), None), + _ => ("", None), + }; + media_block(url, format) +} + +/// `process_file_message`. +fn file_block(part: &Value) -> Result { + let Some(file) = part.get("file").filter(|file| !file.is_null()) else { + return Err(bad_request( + "Content block has type='file' but is missing the required 'file' field", + )); + }; + let source = ["file_id", "file_data"].iter().find_map(|key| { + file.get(*key) + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + }); + let Some(source) = source else { + return Err(bad_request(format!( + "file_data and file_id cannot both be None. Got={part}" + ))); + }; + media_block(source, file.get("format").and_then(Value::as_str)) +} + +/// `_process_document_message`. +fn document_block(part: &Value) -> Result { + let source = part.get("source").cloned().unwrap_or(Value::Null); + let source_type = source + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + if source_type != "base64" { + return Err(bad_request(format!( + "Bedrock Converse only supports base64-encoded document sources, got '{source_type}'. Please convert the document to base64 before sending to Bedrock." + ))); + } + let media_type = source + .get("media_type") + .and_then(Value::as_str) + .unwrap_or_default(); + let data = source + .get("data") + .and_then(Value::as_str) + .unwrap_or_default(); + let format = validate_format(media_type, media_type.split('/').nth(1).unwrap_or_default())?; + let name = format!("Document_{}_{format}", document_hash(data)); + Ok(json!({"document": {"source": {"bytes": data}, "format": format, "name": name}})) +} + +/// `skip_empty_text_blocks` for a list: whitespace-only text parts go. +fn non_empty_parts(parts: &[Value]) -> Vec { + parts + .iter() + .filter(|part| { + !(part.get("type").and_then(Value::as_str) == Some("text") + && part + .get("text") + .and_then(Value::as_str) + .unwrap_or_default() + .trim() + .is_empty()) + }) + .cloned() + .collect() +} + +fn user_blocks(message: &Value, model: &BedrockModel<'_>) -> Result, Error> { + let mut blocks = Vec::new(); + match message.get("content") { + Some(Value::Array(parts)) => { + for part in non_empty_parts(parts) { + if !part.is_object() { + continue; + } + let text = || part.get("text").cloned().unwrap_or(Value::Null); + match part.get("type").and_then(Value::as_str) { + Some("text" | "grounding_source" | "query") => { + blocks.push(json!({"text": text()})) + } + Some("guarded_text") => { + blocks.push(json!({"guardContent": {"text": {"text": text()}}})) + } + Some("image_url") => blocks.push(image_url_block(&part)?), + Some("file") => blocks.push(file_block(&part)?), + Some("document") => blocks.push(document_block(&part)?), + _ => {} + } + blocks.extend(cache_point(part.get("cache_control"), model)); + } + } + Some(Value::String(text)) if !text.is_empty() => { + blocks.push(json!({"text": text})); + blocks.extend(cache_point(message.get("cache_control"), model)); + } + _ => {} + } + Ok(blocks) +} + +/// `_parse_bedrock_tool_result_content_list` item by item. +fn tool_result_parts(content: &[Value]) -> Result, Error> { + let mut parts = Vec::new(); + for item in content { + match item.get("type").and_then(Value::as_str) { + Some("text") => { + parts.push(json!({"text": item.get("text").cloned().unwrap_or(Value::Null)})) + } + Some(kind @ ("image_url" | "file")) => { + let block = match kind { + "image_url" => image_url_block(item)?, + _ => file_block(item)?, + }; + for key in ["image", "document"] { + if let Some(media) = block.get(key) { + parts.push(json!({key: media})); + } + } + } + Some("tool_reference") => { + if let Some(name) = item.get("tool_name").and_then(Value::as_str) { + parts.push(json!({"text": name})); + } + } + _ => {} + } + } + Ok(parts) +} + +/// `_convert_to_bedrock_tool_call_result`. +fn tool_result(message: &Value) -> Result { + let search_results: Vec = message + .get("search_results") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|result| result.is_object()) + .map(|result| json!({"searchResult": result})) + .collect(); + let used_search_results = !search_results.is_empty(); + let content = match used_search_results { + true => search_results, + false => match message.get("content") { + Some(Value::String(text)) => vec![json!({"text": text})], + Some(Value::Array(items)) => tool_result_parts(items)?, + _ => Vec::new(), + }, + }; + let id = match message.get("tool_call_id") { + Some(Value::String(id)) => id.clone(), + Some(Value::Null) => "None".to_owned(), + Some(other) => other.to_string(), + None => uuid::Uuid::new_v4().to_string(), + }; + let mut result = Map::from_iter([ + ("content".to_owned(), Value::Array(content)), + ("toolUseId".to_owned(), json!(sanitize_tool_use_id(&id))), + ]); + if used_search_results { + result.insert("status".into(), json!("success")); + } + Ok(json!({"toolResult": result})) +} + +/// `split_concatenated_json_objects`. +fn concatenated_objects(raw: &str) -> Vec { + serde_json::Deserializer::from_str(raw.trim()) + .into_iter::() + .map_while(Result::ok) + .map(|value| match value { + Value::Object(_) => value, + _ => json!({}), + }) + .collect() +} + +/// `_convert_to_bedrock_tool_call_invoke`. +fn tool_uses(calls: &[Value], model: &BedrockModel<'_>) -> Result, Error> { + let mut blocks = Vec::new(); + for call in calls.iter().filter(|call| call.get("function").is_some()) { + let id = call.get("id").and_then(Value::as_str).unwrap_or_default(); + let function = &call["function"]; + let name = make_valid_bedrock_tool_name( + function + .get("name") + .and_then(Value::as_str) + .unwrap_or_default(), + ); + let arguments = function + .get("arguments") + .and_then(Value::as_str) + .unwrap_or_default(); + let cache = cache_point(call.get("cache_control"), model); + if arguments.trim().is_empty() { + blocks.push(json!({"toolUse": {"input": {}, "name": name, "toolUseId": sanitize_tool_use_id(id)}})); + } else { + match serde_json::from_str::(arguments) { + Ok(parsed) => { + let input = match parsed { + Value::Object(_) => parsed, + _ => json!({}), + }; + blocks.push(json!({"toolUse": {"input": input, "name": name, "toolUseId": sanitize_tool_use_id(id)}})); + } + Err(_) => { + let objects = concatenated_objects(arguments); + if objects.is_empty() { + blocks.push(json!({"toolUse": {"input": {}, "name": name, "toolUseId": sanitize_tool_use_id(id)}})); + } else { + for (index, object) in objects.into_iter().enumerate() { + let block_id = match index { + 0 => id.to_owned(), + _ => format!("{id}_{index}"), + }; + blocks.push(json!({"toolUse": {"input": object, "name": name, "toolUseId": sanitize_tool_use_id(&block_id)}})); + } + } + } + } + } + blocks.extend(cache); + } + Ok(blocks) +} + +/// `translate_thinking_blocks_to_reasoning_content_blocks` and +/// `add_thinking_blocks_to_assistant_content`. +fn add_thinking_blocks(blocks: &[Value], to_claude: bool, out: &mut Vec) { + let mut reasoning = Vec::new(); + for block in blocks { + if to_claude && crate::anthropic::chat::messages::is_encrypted_reasoning(block) { + continue; + } + let thinking = block + .get("thinking") + .and_then(Value::as_str) + .unwrap_or_default(); + let data = block + .get("data") + .and_then(Value::as_str) + .filter(|d| !d.trim().is_empty()); + if block.get("type").and_then(Value::as_str) == Some("redacted_thinking") + && to_claude + && let Some(data) = data + { + use base64::Engine; + let content = base64::engine::general_purpose::STANDARD + .encode(native_redacted_thinking_data(data).as_bytes()); + reasoning.push(json!({"reasoningContent": {"redactedContent": content}})); + continue; + } + match block.get("signature").filter(|s| !s.is_null()) { + Some(signature) if !signature.as_str().is_some_and(str::is_empty) => reasoning.push( + json!({"reasoningContent": {"reasoningText": {"text": thinking, "signature": signature}}}), + ), + _ => { + if !thinking.trim().is_empty() { + out.push(json!({"text": thinking})); + } + } + } + } + out.extend(reasoning); +} + +fn assistant_blocks(message: &Value, model: &BedrockModel<'_>) -> Result, Error> { + let mut blocks = Vec::new(); + let to_claude = model.accepts_redacted_thinking(); + if let Some(Value::Array(thinking)) = message.get("thinking_blocks") { + add_thinking_blocks(thinking, to_claude, &mut blocks); + } + match message.get("content") { + Some(Value::Array(parts)) => { + let parts = non_empty_parts(parts); + let mut converted = Vec::new(); + for part in parts.iter().filter(|part| part.is_object()) { + match part.get("type").and_then(Value::as_str) { + Some("thinking" | "redacted_thinking") => { + add_thinking_blocks(std::slice::from_ref(part), to_claude, &mut converted) + } + Some("text") => { + let text = part.get("text").and_then(Value::as_str).unwrap_or_default(); + if !text.trim().is_empty() { + converted.push(json!({"text": text})); + } + } + Some("image_url") => converted.push(image_url_block(part)?), + _ => {} + } + converted.extend(cache_point(part.get("cache_control"), model)); + } + blocks.extend(converted); + } + Some(Value::String(text)) => { + let non_content = message.get("tool_calls").is_some_and(truthy) + || message.get("function_call").is_some_and(truthy); + if !text.trim().is_empty() { + blocks.push(json!({"text": text})); + } + // An all-whitespace string becomes `None` when the message carries tool calls, + // and a `None` content adds no cache point. + if !(text.trim().is_empty() && non_content) { + blocks.extend(cache_point(message.get("cache_control"), model)); + } + } + _ => {} + } + if let Some(Value::Array(calls)) = message.get("tool_calls") + && !calls.is_empty() + { + blocks.extend(tool_uses(calls, model)?); + } + Ok(blocks) +} + +/// `_deduplicate_bedrock_content_blocks`. +fn dedupe(blocks: Vec, key: &str) -> Vec { + let mut seen: HashSet = HashSet::new(); + blocks + .into_iter() + .filter(|block| { + match block + .get(key) + .and_then(|keyed| keyed.get("toolUseId")) + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + { + Some(id) => seen.insert(id.to_owned()), + None => true, + } + }) + .collect() +} + +/// `_sort_bedrock_assistant_content_blocks`. +fn sorted_assistant_blocks(mut blocks: Vec) -> Vec { + blocks.sort_by_key(|block| { + if block.get("reasoningContent").is_some() { + 0 + } else if block.get("toolUse").is_some() || block.get("cachePoint").is_some() { + 2 + } else { + 1 + } + }); + blocks +} + +fn push_user_turn(contents: &mut Vec, blocks: Vec) { + if let Some(last) = contents.last_mut() + && last["role"] == "user" + && let Some(Value::Array(existing)) = last.get_mut("content") + { + existing.extend(blocks); + return; + } + contents.push(json!({"role": "user", "content": blocks})); +} + +/// `_bedrock_converse_messages_pt`. +pub fn converse_messages( + messages: &[Value], + model: &BedrockModel<'_>, +) -> Result, Error> { + if messages.is_empty() { + return Err(bad_request(format!( + "{BAD_MESSAGE}bedrock requires at least one non-system message" + ))); + } + let mut contents: Vec = Vec::new(); + let mut index = 0; + while index < messages.len() { + let start = index; + let mut user_content = Vec::new(); + while index < messages.len() && role(&messages[index]) == "user" { + user_content.extend(user_blocks(&messages[index], model)?); + index += 1; + } + if !user_content.is_empty() { + push_user_turn(&mut contents, user_content); + } + let mut tool_content = Vec::new(); + while index < messages.len() && role(&messages[index]) == "tool" { + let message = &messages[index]; + tool_content.push(tool_result(message)?); + let control = message + .get("cache_control") + .filter(|c| !c.is_null()) + .or_else(|| { + message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .find_map(|part| part.get("cache_control").filter(|c| !c.is_null())) + }); + tool_content.extend(cache_point(control, model)); + index += 1; + } + let tool_content = dedupe(tool_content, "toolResult"); + if !tool_content.is_empty() { + push_user_turn(&mut contents, tool_content); + } + let mut assistant_content = Vec::new(); + while index < messages.len() && role(&messages[index]) == "assistant" { + assistant_content.extend(assistant_blocks(&messages[index], model)?); + index += 1; + } + let assistant_content = sorted_assistant_blocks(dedupe(assistant_content, "toolUse")); + if !assistant_content.is_empty() { + contents.push(json!({"role": "assistant", "content": assistant_content})); + } + if index == start { + return Err(bad_request(format!( + "{BAD_MESSAGE}passed in {}", + crate::base_llm::chat::params::python_dumps(&messages[index]) + ))); + } + } + rename_duplicate_documents(&mut contents); + let mut contents: Vec = contents + .into_iter() + .map(with_text_when_document_only) + .collect(); + if model.accepts_redacted_thinking() + && let Some(last) = contents.last_mut() + && last["role"] == "assistant" + && let Some(Value::Array(blocks)) = last.get_mut("content") + && let Some(position) = blocks.iter().rposition(|block| block.get("text").is_some()) + { + let text = blocks[position]["text"] + .as_str() + .unwrap_or_default() + .trim_end() + .to_owned(); + blocks[position] = json!({"text": text}); + } + if !model.supports_tool_result_images() { + contents = contents + .into_iter() + .map(images_beside_tool_results) + .collect(); + } + Ok(contents) +} + +/// `_rename_duplicate_bedrock_document_names`. +fn rename_duplicate_documents(contents: &mut [Value]) { + let names = |contents: &[Value]| -> Vec { + contents + .iter() + .flat_map(|message| message["content"].as_array().cloned().unwrap_or_default()) + .filter_map(|block| { + block + .get("document")? + .get("name")? + .as_str() + .map(str::to_owned) + }) + .filter(|name| !name.is_empty()) + .collect() + }; + let mut used: HashSet = names(contents).into_iter().collect(); + let mut counts: std::collections::HashMap = std::collections::HashMap::new(); + for message in contents.iter_mut() { + let Some(Value::Array(blocks)) = message.get_mut("content") else { + continue; + }; + for block in blocks { + let Some(Value::Object(document)) = block.get_mut("document") else { + continue; + }; + let Some(name) = document + .get("name") + .and_then(Value::as_str) + .map(str::to_owned) + else { + continue; + }; + if name.is_empty() { + continue; + } + let count = counts.entry(name.clone()).or_insert(0); + *count += 1; + if *count > 1 { + let mut suffix = *count; + let mut renamed = format!("{name}_{suffix}"); + while used.contains(&renamed) { + suffix += 1; + renamed = format!("{name}_{suffix}"); + } + used.insert(renamed.clone()); + document.insert("name".into(), json!(renamed)); + } + } + } +} + +/// `_with_text_when_document_only`. +fn with_text_when_document_only(message: Value) -> Value { + let blocks = message["content"].as_array().cloned().unwrap_or_default(); + let needs_text = message["role"] == "user" + && blocks.iter().any(|block| block.get("document").is_some()) + && blocks.iter().all(|block| block.get("text").is_none()); + if !needs_text { + return message; + } + let cut = match blocks + .last() + .is_some_and(|block| block.get("cachePoint").is_some()) + { + true => blocks.len() - 1, + false => blocks.len(), + }; + let mut content = blocks[..cut].to_vec(); + content.push(json!({"text": DOCUMENT_PLACEHOLDER_TEXT})); + content.extend(blocks[cut..].iter().cloned()); + json!({"role": "user", "content": content}) +} + +/// `place_tool_result_images` for a model that refuses an image inside a tool result. +fn images_beside_tool_results(message: Value) -> Value { + let Some(blocks) = message["content"].as_array() else { + return message; + }; + let mut rewritten = Vec::new(); + let mut changed = false; + for block in blocks { + let Some(result) = block.get("toolResult") else { + rewritten.push(block.clone()); + continue; + }; + let parts = result["content"].as_array().cloned().unwrap_or_default(); + let images: Vec = parts + .iter() + .filter_map(|part| part.get("image").cloned()) + .collect(); + if images.is_empty() { + rewritten.push(block.clone()); + continue; + } + changed = true; + let mut kept: Vec = parts + .into_iter() + .filter(|part| part.get("image").is_none()) + .collect(); + if kept.is_empty() { + kept.push(json!({"text": TOOL_RESULT_IMAGE_PLACEHOLDER})); + } + let mut result = result.clone(); + result["content"] = Value::Array(kept); + rewritten.push(json!({"toolResult": result})); + rewritten.extend(images.into_iter().map(|image| json!({"image": image}))); + } + match changed { + true => json!({"role": message["role"].clone(), "content": rewritten}), + false => message, + } +} + +/// `normalize_json_schema_custom_types_to_object`. +fn normalize_custom_types(schema: &mut Value) { + let Value::Object(node) = schema else { return }; + if node.get("type").and_then(Value::as_str) == Some("custom") { + node.insert("type".into(), json!("object")); + } + for key in ["items", "additionalProperties"] { + if let Some(child @ Value::Object(_)) = node.get_mut(key) { + normalize_custom_types(child); + } + } + for key in ["properties", "$defs", "definitions"] { + if let Some(Value::Object(children)) = node.get_mut(key) { + children.values_mut().for_each(normalize_custom_types); + } + } + for key in ["anyOf", "allOf", "oneOf"] { + if let Some(Value::Array(children)) = node.get_mut(key) { + children.iter_mut().for_each(normalize_custom_types); + } + } +} + +/// `_bedrock_tools_pt`, with the names it rewrote mapped back for the response. +pub fn converse_tools( + tools: &[Value], + model: &BedrockModel<'_>, +) -> Result<(Vec, std::collections::HashMap), Error> { + let strict_supported = model.supports_strict_tools(); + let mut blocks = Vec::new(); + let mut renamed = std::collections::HashMap::new(); + for (index, tool) in tools.iter().enumerate() { + let Some(object) = tool.as_object() else { + continue; + }; + if ["systemTool", "toolSpec", "cachePoint"] + .iter() + .any(|key| object.contains_key(*key)) + { + blocks.push(tool.clone()); + continue; + } + let kind = object + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + if kind.starts_with("computer_") { + return Err(Error::Unsupported( + "a computer use tool on Bedrock Converse", + )); + } + if matches!( + kind, + "tool_search_tool_regex_20251119" | "tool_search_tool_bm25_20251119" + ) { + continue; + } + if !object.contains_key("function") && !object.contains_key("input_schema") { + continue; + } + let default_schema = json!({"type": "object", "properties": {}}); + let (mut parameters, raw_name, description) = + if object.contains_key("input_schema") && !object.contains_key("function") { + ( + object + .get("input_schema") + .filter(|v| truthy(v)) + .cloned() + .unwrap_or(default_schema), + object + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + object.get("description").cloned(), + ) + } else { + let function = object.get("function").cloned().unwrap_or(Value::Null); + ( + function + .get("parameters") + .cloned() + .unwrap_or(default_schema), + function + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + function.get("description").cloned(), + ) + }; + let raw_name = match raw_name.trim().is_empty() { + true => format!("litellm_unnamed_tool_{index}"), + false => raw_name, + }; + let name = make_valid_bedrock_tool_name(&raw_name); + if name != raw_name { + renamed.insert(name.clone(), raw_name.clone()); + } + let description = match description { + Some(Value::String(text)) if !text.is_empty() => json!(text), + Some(other) if truthy(&other) => other, + _ => json!(name), + }; + let defs = match parameters.as_object_mut().and_then(|p| p.remove("$defs")) { + Some(Value::Object(defs)) => defs, + _ => Map::new(), + }; + unpack_defs(&mut parameters, &defs); + normalize_custom_types(&mut parameters); + let valid_roots = [ + "array", "boolean", "integer", "null", "number", "object", "string", + ]; + if !parameters + .get("type") + .and_then(Value::as_str) + .is_some_and(|kind| valid_roots.contains(&kind)) + && let Value::Object(object) = &mut parameters + { + object.insert("type".into(), json!("object")); + } + let mut json_schema = Map::from_iter([ + ( + "type".to_owned(), + parameters.get("type").cloned().unwrap_or(Value::Null), + ), + ( + "properties".to_owned(), + parameters + .get("properties") + .cloned() + .unwrap_or_else(|| json!({})), + ), + ( + "required".to_owned(), + parameters + .get("required") + .cloned() + .unwrap_or_else(|| json!([])), + ), + ]); + if strict_supported + && let Some(additional) = parameters + .get("additionalProperties") + .filter(|v| !v.is_null()) + { + json_schema.insert("additionalProperties".into(), additional.clone()); + } + let mut spec = Map::from_iter([ + ("inputSchema".to_owned(), json!({"json": json_schema})), + ("name".to_owned(), json!(name)), + ("description".to_owned(), description), + ]); + let strict = object.get("function").and_then(|f| f.get("strict")); + if strict_supported && strict.is_some_and(truthy) { + spec.insert("strict".into(), strict.cloned().unwrap_or(Value::Null)); + } + blocks.push(json!({"toolSpec": spec})); + if let Some(control) = object.get("cache_control").filter(|c| !c.is_null()) + && model.accepts_cache_points() + && control + .get("type") + .and_then(Value::as_str) + .unwrap_or("ephemeral") + == "ephemeral" + { + blocks.push(json!({"cachePoint": cache_point_block(Some(control), model)})); + } + } + Ok((blocks, renamed)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::base_llm::chat::params::ChatRequestContext; + + fn model() -> BedrockModel<'static> { + BedrockModel::new( + "anthropic.claude-sonnet-4-5-v1:0", + &ChatRequestContext { + drop_params: Some(false), + model_info: None, + }, + ) + } + + #[rstest] + #[case("call_1", "call_1")] + #[case("call 1", "call_1_")] + fn tool_use_ids_are_sanitized(#[case] id: &str, #[case] prefix: &str) { + let sanitized = sanitize_tool_use_id(id); + assert!(sanitized.starts_with(prefix)); + if id != prefix { + assert_eq!(sanitized.len(), prefix.len() + TOOL_USE_ID_HASH_LEN); + } + } + + #[rstest] + fn a_tool_round_trip_becomes_tool_use_and_tool_result() { + let messages = vec![ + json!({"role": "user", "content": "weather?"}), + json!({"role": "assistant", "content": "checking", "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "get.weather", "arguments": "{\"city\":\"Paris\"}"}} + ]}), + json!({"role": "tool", "tool_call_id": "c1", "content": "sunny"}), + ]; + let converted = converse_messages(&messages, &model()).unwrap(); + assert_eq!( + converted, + vec![ + json!({"role": "user", "content": [{"text": "weather?"}]}), + json!({"role": "assistant", "content": [ + {"text": "checking"}, + {"toolUse": {"input": {"city": "Paris"}, "name": "get_weather", "toolUseId": "c1"}} + ]}), + json!({"role": "user", "content": [{"toolResult": {"content": [{"text": "sunny"}], "toolUseId": "c1"}}]}), + ] + ); + } + + #[rstest] + fn images_and_documents_become_blocks() { + let messages = vec![json!({"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBE"}, "cache_control": {"type": "ephemeral"}} + ]})]; + let converted = converse_messages(&messages, &model()).unwrap(); + let content = converted[0]["content"].as_array().unwrap(); + assert_eq!( + content[0], + json!({"image": {"source": {"bytes": "AAAA"}, "format": "png"}}) + ); + assert_eq!(content[1]["document"]["format"], "pdf"); + assert!( + content[1]["document"]["name"] + .as_str() + .unwrap() + .starts_with("DocumentPDFmessages_") + ); + assert_eq!(content[2], json!({"text": "."})); + assert_eq!(content[3], json!({"cachePoint": {"type": "default"}})); + } + + #[rstest] + fn system_messages_become_system_blocks_and_operator_notes() { + let (messages, system) = transform_system_message( + vec![ + json!({"role": "system", "content": "be terse", "cache_control": {"type": "ephemeral"}}), + json!({"role": "user", "content": "hi"}), + json!({"role": "system", "content": "later"}), + ], + &model(), + ); + assert_eq!( + system, + vec![ + json!({"text": "be terse"}), + json!({"cachePoint": {"type": "default"}}) + ] + ); + assert_eq!(messages[1]["content"][0]["text"], CONVERTED_SYSTEM_NOTE); + assert_eq!(messages[1]["content"][1]["text"], "later"); + } + + #[rstest] + fn orphaned_tool_blocks_become_text() { + let messages = neutralize_orphaned_tool_blocks( + vec![ + json!({"role": "assistant", "tool_calls": [{"id": "c", "function": {"name": "f", "arguments": "{}"}}]}), + json!({"role": "tool", "tool_call_id": "c", "content": "ok"}), + ], + false, + false, + ); + assert_eq!( + messages[0], + json!({"role": "assistant", "content": "[tool call c: f({})]"}) + ); + assert_eq!( + messages[1], + json!({"role": "user", "content": "[tool result for c: ok]"}) + ); + } + + #[rstest] + fn tools_become_tool_specs() { + let (tools, renamed) = converse_tools( + &[json!({"type": "function", "function": {"name": "get.weather", "parameters": { + "type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"] + }}})], + &model(), + ) + .unwrap(); + assert_eq!( + tools, + vec![ + json!({"toolSpec": {"inputSchema": {"json": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}, "name": "get_weather", "description": "get_weather"}}) + ] + ); + assert_eq!(renamed["get_weather"], "get.weather"); + } +} diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_params.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_params.rs new file mode 100644 index 00000000000..04c44d8f16c --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_params.rs @@ -0,0 +1,1039 @@ +//! Python's `AmazonConverseConfig.get_supported_openai_params` and `map_openai_params`, and +//! the Bedrock model-id reads behind them. + +use serde_json::{Map, Value, json}; + +use crate::{ + Error, ErrorDetail, + anthropic::chat::params::{ + ModelCapabilities, apply_sampling_param, cap_thinking_budget, map_context_management, + map_reasoning_effort, output_config_effort, python_str, translate_legacy_thinking, + unmapped_reasoning_effort, + }, + base_llm::chat::{ + params::{ + ChatRequestContext, DEFAULT_MAX_TOKENS, DropPolicy, RESPONSE_FORMAT_TOOL_NAME, + capability, is_thinking_enabled, model_field, ordered, python_dumps, unsupported_value, + }, + response_format::ResponseFormatSchema, + }, +}; + +/// `get_bedrock_cross_region_inference_regions`. +const CROSS_REGION_PREFIXES: [&str; 7] = ["global", "us", "eu", "apac", "jp", "au", "us-gov"]; + +/// `BEDROCK_MIN_THINKING_BUDGET_TOKENS`. +const MIN_THINKING_BUDGET_TOKENS: i64 = 1024; + +/// `BEDROCK_OPENAI_COMPAT_MIN_MAX_TOKENS`. +const OPENAI_COMPAT_MIN_MAX_TOKENS: i64 = 16; + +/// `_BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER`. +const OUTPUT_CONFIG_EFFORT_ORDER: [&str; 5] = ["low", "medium", "high", "xhigh", "max"]; + +/// `strip_bedrock_routing_prefix`: each prefix in turn, as Python applies them. +pub fn strip_bedrock_routing_prefix(model: &str) -> String { + let mut model = model.to_owned(); + for prefix in [ + "bedrock/", + "chat_completions/", + "converse/", + "invoke/", + "openai/", + "mantle/", + "nova-2/", + "nova/", + ] { + if model.starts_with(prefix) { + model = model + .split_once('/') + .map(|(_, rest)| rest.to_owned()) + .unwrap_or_default(); + } + } + model +} + +/// `strip_bedrock_throughput_suffix`: `::k` loses its throughput and `[...]` is cut. +fn strip_throughput_suffix(model: &str) -> String { + let mut model = model.to_owned(); + if let Some(head) = model.strip_suffix('k') + && let Some((rest, digits)) = head.rsplit_once(':') + && !digits.is_empty() + && digits.chars().all(|c| c.is_ascii_digit()) + && let Some((_, minor)) = rest.rsplit_once(':') + && !minor.is_empty() + && minor.chars().all(|c| c.is_ascii_digit()) + { + model = rest.to_owned(); + } + if model.ends_with(']') + && let Some(open) = model.rfind('[') + && model[open + 1..model.len() - 1] + .chars() + .all(|c| c.is_alphanumeric() || c == '_') + && open + 1 < model.len() - 1 + { + model.truncate(open); + } + model +} + +/// Whether `segment` reads as an AWS region, such as `us-east-1` or `us-gov-west-1`. +fn is_aws_region(segment: &str) -> bool { + let parts: Vec<&str> = segment.split('-').collect(); + parts.len() >= 3 + && parts[0].len() == 2 + && parts[0].chars().all(|c| c.is_ascii_lowercase()) + && parts + .last() + .is_some_and(|last| !last.is_empty() && last.chars().all(|c| c.is_ascii_digit())) + && parts[1..parts.len() - 1] + .iter() + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_lowercase())) +} + +/// `get_bedrock_base_model`. +pub fn bedrock_base_model(model: &str) -> String { + let mut stripped = model; + for prefix in ["bedrock/converse/", "bedrock/", "converse/"] { + if let Some(rest) = stripped.strip_prefix(prefix) { + stripped = rest; + break; + } + } + if stripped.starts_with("nova-2/") { + return "amazon.nova-2-custom".into(); + } + if stripped.starts_with("nova/") { + return "amazon.nova-custom".into(); + } + let mut model = strip_bedrock_routing_prefix(model); + if model.to_lowercase().contains("arn") { + model = model.rsplit('/').next().unwrap_or_default().to_owned(); + } + let model = strip_throughput_suffix(&model); + if let Some((region, rest)) = model.split_once('.') + && CROSS_REGION_PREFIXES.contains(®ion) + { + return rest.to_owned(); + } + if let Some((region, rest)) = model.split_once('/') + && is_aws_region(region) + { + return rest.to_owned(); + } + model +} + +/// `_is_nova_2_model`. +fn is_nova_2_model(model: &str) -> bool { + let mut stripped = model; + for prefix in ["bedrock/converse/", "bedrock/", "converse/"] { + if let Some(rest) = stripped.strip_prefix(prefix) { + stripped = rest; + break; + } + } + for prefix in ["us.", "eu.", "apac."] { + if let Some(rest) = stripped.strip_prefix(prefix) { + stripped = rest; + break; + } + } + stripped.starts_with("amazon.nova-2-") || stripped.starts_with("nova-2/") +} + +/// `openai.gpt-` anywhere in the id. +fn is_openai_gpt_reasoning_model(model: &str) -> bool { + model.match_indices("openai.gpt-").any(|(at, _)| { + model[at + "openai.gpt-".len()..] + .chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) + }) +} + +/// `xai.grok-` after a start, `.` or `/`. +fn is_xai_grok_model(model: &str) -> bool { + model + .match_indices("xai.grok-") + .any(|(at, _)| at == 0 || matches!(model.as_bytes()[at - 1], b'.' | b'/')) +} + +/// `openai.gpt-[.]` after a start, `.` or `/`. +fn openai_gpt_version(model: &str) -> Option<(u32, u32)> { + model.match_indices("openai.").find_map(|(at, _)| { + if at != 0 && !matches!(model.as_bytes()[at - 1], b'.' | b'/') { + return None; + } + let rest = model[at..].strip_prefix("openai.gpt-")?; + let major: String = rest.chars().take_while(char::is_ascii_digit).collect(); + if major.is_empty() || major.len() > 3 { + return None; + } + let after = &rest[major.len()..]; + let minor = after + .strip_prefix('.') + .map(|tail| { + tail.chars() + .take_while(char::is_ascii_digit) + .collect::() + }) + .filter(|minor| !minor.is_empty() && minor.len() <= 3) + .unwrap_or_default(); + Some((major.parse().ok()?, minor.parse().unwrap_or(0))) + }) +} + +/// `bedrock_rejects_stop_sequences`. +fn rejects_stop_sequences(model: &str) -> bool { + is_xai_grok_model(model) || openai_gpt_version(model).is_some_and(|version| version >= (5, 6)) +} + +/// The model a Converse request is for, and the cost-map reads its rules make. +pub struct BedrockModel<'a> { + pub model: String, + pub base: String, + pub context: ChatRequestContext<'a>, + pub caps: ModelCapabilities<'a>, +} + +impl<'a> BedrockModel<'a> { + pub fn new(model: &str, context: &ChatRequestContext<'a>) -> Self { + Self { + model: model.to_owned(), + base: bedrock_base_model(model), + context: *context, + caps: ModelCapabilities::new(model, context), + } + } + + pub fn is_anthropic(&self) -> bool { + self.base.starts_with("anthropic") + } + + /// The keys Python's provider-aware lookups resolve `model` through. + pub fn candidates(&self) -> Vec { + let stripped = strip_bedrock_routing_prefix(&self.model); + let mut candidates = vec![ + self.model.clone(), + format!("bedrock/{}", self.model), + format!("bedrock_converse/{}", self.model), + stripped.clone(), + self.base.clone(), + format!("bedrock/{}", self.base), + ]; + candidates.dedup(); + candidates + } + + pub fn flag(&self, key: &str) -> Result, Error> { + capability(&self.context, &self.candidates(), key) + } + + pub fn supports(&self, key: &str) -> Result { + Ok(self.flag(key)? == Some(true)) + } + + /// `_supports_sampling_params`. + pub fn supports_sampling(&self) -> Result { + if self.is_anthropic() { + return Ok(true); + } + let mut keys = vec![self.model.clone()]; + keys.extend(["global.", "us.", "eu."].map(|prefix| format!("{prefix}{}", self.base))); + Ok(capability(&self.context, &keys, "supports_sampling_params")?.unwrap_or(true)) + } + + fn is_deepseek(&self) -> bool { + self.model.contains("deepseek") || self.base.contains("deepseek") + } + + fn is_deepseek_r1(&self) -> bool { + self.model.contains("deepseek.r1") || self.base.contains("deepseek.r1") + } + + /// `_model_accepts_anthropic_thinking_param`. + fn accepts_anthropic_thinking(&self) -> Result { + if self.is_deepseek() { + return Ok(false); + } + Ok(self.model.contains("claude-3-7") + || self.model.contains("claude-sonnet-4") + || self.model.contains("claude-opus-4") + || self.supports("supports_reasoning")?) + } + + /// `bedrock_converse_supports_parallel_tool_use_config`. + pub fn supports_parallel_tool_use_config(&self) -> Result { + capability( + &self.context, + &[self.model.clone(), self.base.clone()], + "supports_parallel_tool_use_config", + ) + .map(|flag| flag == Some(true)) + } + + /// `bedrock_model_accepts_cache_points`. + pub fn accepts_cache_points(&self) -> bool { + let openai_family = self + .model + .match_indices("openai.") + .any(|(at, _)| at == 0 || matches!(self.model.as_bytes()[at - 1], b'.' | b'/')); + if openai_family { + return false; + } + let Some(info) = self.context.model_info else { + return true; + }; + let map_keys = [self.model.clone(), self.base.clone()]; + let registered = ["", "converse/", "converse_like/"] + .map(|route| format!("bedrock/{route}{}", self.model)); + let explicit = registered + .iter() + .chain(map_keys.iter()) + .filter_map(|key| info.entry(key)) + .find_map(|entry| { + entry + .get("supports_prompt_cache_breakpoint") + .filter(|flag| !flag.is_null()) + .map(|flag| flag == &Value::Bool(true)) + }); + if let Some(explicit) = explicit { + return explicit; + } + let entries: Vec<_> = map_keys.iter().filter_map(|key| info.entry(key)).collect(); + entries.is_empty() + || entries + .iter() + .any(|entry| entry.get("supports_prompt_caching") == Some(&Value::Bool(true))) + } + + /// `supports_bedrock_prompt_cache_ttl`. + pub fn cache_ttl_supported(&self) -> bool { + model_field( + &self.context, + &[self.model.clone(), self.base.clone()], + "cache_creation_input_token_cost_above_1hr", + ) + .is_some() + } + + /// `bedrock_converse_supports_strict_tools`. + pub fn supports_strict_tools(&self) -> bool { + if !self.is_anthropic() { + return false; + } + self.context + .model_info + .and_then(|info| { + [self.base.clone(), format!("bedrock/{}", self.base)] + .iter() + .filter_map(|key| info.entry(key)) + .find_map(|entry| { + entry + .get("bedrock_converse_supports_strict_tools") + .and_then(Value::as_bool) + }) + }) + .unwrap_or(true) + } + + /// `bedrock_converse_supports_tool_result_images`. + pub fn supports_tool_result_images(&self) -> bool { + let Some(info) = self.context.model_info else { + return true; + }; + let keys = [ + self.model.clone(), + strip_bedrock_routing_prefix(&self.model), + ]; + keys.iter().filter_map(|key| info.entry(key)).all(|entry| { + entry.get("supports_bedrock_converse_tool_result_images") != Some(&Value::Bool(false)) + }) + } + + /// `bedrock_converse_accepts_redacted_thinking` for a model id that names its model. + pub fn accepts_redacted_thinking(&self) -> bool { + self.base.starts_with("anthropic") || self.base.starts_with("claude") + } + + /// `normalize_bedrock_opus_output_config_effort`: an effort above the model's ceiling + /// comes down to it. + pub fn normalize_output_config_effort(&self, output_config: &mut Map) { + let Some(effort) = output_config.get("effort").and_then(Value::as_str) else { + return; + }; + let Some(rank) = OUTPUT_CONFIG_EFFORT_ORDER.iter().position(|e| *e == effort) else { + return; + }; + let ceiling = model_field( + &self.context, + &self.candidates(), + "bedrock_output_config_effort_ceiling", + ); + let Some(ceiling) = ceiling.as_ref().and_then(Value::as_str) else { + return; + }; + if let Some(ceiling_rank) = OUTPUT_CONFIG_EFFORT_ORDER + .iter() + .position(|e| *e == ceiling) + && rank > ceiling_rank + { + output_config.insert("effort".into(), json!(ceiling)); + } + } + + /// `_validate_anthropic_adaptive_effort`. + pub fn validate_adaptive_effort(&self, effort: &Value) -> Result<(), Error> { + let text = effort + .as_str() + .filter(|effort| OUTPUT_CONFIG_EFFORT_ORDER.contains(effort)); + let Some(text) = text else { + let shown = match effort { + Value::String(text) => format!("'{text}'"), + other => python_dumps(other), + }; + return Err(unsupported_value(format!( + "Invalid reasoning_effort/output_config.effort value: {shown}. Must be one of: 'low', 'medium', 'high', 'xhigh', or 'max'." + ))); + }; + let refused = match text { + "max" + if !(self.caps.is_adaptive()? + || self.caps.supports("supports_max_reasoning_effort")?) => + { + Some("max") + } + "xhigh" if !self.caps.supports("supports_xhigh_reasoning_effort")? => Some("xhigh"), + _ => None, + }; + match refused { + Some(value) => Err(Error::InvalidRequest(ErrorDetail::UnsupportedValue { + field: "effort", + value, + model: self.model.clone(), + })), + None => Ok(()), + } + } + + /// `AnthropicConfig.model_supports_effort_param` under `bedrock`. + pub fn supports_effort_param(&self) -> Result { + if self.caps.supports("supports_output_config")? { + return Ok(true); + } + for level in ["low", "minimal", "medium", "high", "xhigh", "max"] { + if self + .caps + .supports(&format!("supports_{level}_reasoning_effort"))? + { + return Ok(true); + } + } + Ok(false) + } +} + +/// `get_supported_openai_params`. +pub fn supported_params(model: &BedrockModel<'_>) -> Result, Error> { + let mut params = vec![ + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "stop", + ]; + if model.supports_sampling()? { + params.extend(["temperature", "top_p"]); + } + params.extend([ + "extra_headers", + "response_format", + "requestMetadata", + "service_tier", + "parallel_tool_calls", + ]); + if model.model.contains("arn") { + params.extend([ + "tools", + "tool_choice", + "thinking", + "reasoning_effort", + "output_config", + ]); + if model.model.to_lowercase().contains("nova") { + params.push("web_search_options"); + } + return Ok(params); + } + let base = model.base.as_str(); + if [ + "anthropic", + "mistral", + "cohere", + "meta.llama3-1", + "meta.llama3-2", + "meta.llama3-3", + "meta.llama4", + "amazon.nova", + ] + .iter() + .any(|family| base.starts_with(family)) + || model.supports("supports_function_calling")? + { + params.push("tools"); + } + if base.starts_with("amazon.nova") { + params.push("web_search_options"); + } + if model.supports("supports_tool_choice")? { + params.push("tool_choice"); + } + if model.model.contains("gpt-oss") + || is_openai_gpt_reasoning_model(&model.model) + || is_openai_gpt_reasoning_model(base) + { + params.push("reasoning_effort"); + } else if model.is_deepseek() { + if !model.is_deepseek_r1() { + params.push("reasoning_effort"); + } + } else if is_nova_2_model(&model.model) { + params.push("reasoning_effort"); + } else if model.accepts_anthropic_thinking()? { + params.extend(["thinking", "reasoning_effort", "output_config"]); + } + if model.is_anthropic() { + params.push("context_management"); + } + Ok(params) +} + +/// `map_tool_choice_values`. +pub fn map_tool_choice_values( + model: &BedrockModel<'_>, + tool_choice: &Value, + drop: DropPolicy, +) -> Result, Error> { + // `forced_tool_use_downgraded`: refused without `drop_params`, `auto` with it. + let downgraded = || -> Result { + if !model.caps.forced_tool_use_unsupported()? { + return Ok(false); + } + drop.unsupported(|| forced_tool_use_refusal(&model.model))?; + Ok(true) + }; + match tool_choice { + Value::String(choice) if choice == "none" => { + drop.unsupported(|| { + unsupported_value( + "Bedrock doesn't support tool_choice=none. To drop it from the call, set `litellm.drop_params = True.", + ) + })?; + Ok(None) + } + Value::String(choice) if choice == "required" => Ok(Some(match downgraded()? { + true => json!({"auto": {}}), + false => json!({"any": {}}), + })), + Value::String(choice) if choice == "auto" => Ok(Some(json!({"auto": {}}))), + Value::Object(choice) => { + if downgraded()? { + return Ok(Some(json!({"auto": {}}))); + } + let name = choice + .get("function") + .and_then(|function| function.get("name")) + .and_then(Value::as_str) + .unwrap_or_default(); + Ok(Some( + json!({"tool": {"name": make_valid_bedrock_tool_name(name)}}), + )) + } + other => Err(unsupported_value(format!( + "Bedrock doesn't support tool_choice={}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.", + python_str(other) + ))), + } +} + +fn forced_tool_use_refusal(model: &str) -> Error { + unsupported_value(format!( + "{model} does not support forced tool use (tool_choice='required' or a named tool). Use tool_choice='auto' and tell the model in the prompt when to call the tool, or set `litellm.drop_params = True` to downgrade to 'auto' automatically." + )) +} + +/// `make_valid_bedrock_tool_name`: `[a-zA-Z][a-zA-Z0-9_-]*`. +pub fn make_valid_bedrock_tool_name(name: &str) -> String { + let Some(first) = name.chars().next() else { + return String::new(); + }; + let prefixed = match first.is_alphabetic() { + true => name.to_owned(), + false => format!("a{name}"), + }; + prefixed + .chars() + .map(|c| match c.is_alphanumeric() || c == '_' || c == '-' { + true => c, + false => '_', + }) + .collect() +} + +/// `_add_additional_properties_to_schema`. +fn with_closed_objects(schema: &Value) -> Value { + let Value::Object(object) = schema else { + return schema.clone(); + }; + let mut result = object.clone(); + if result.get("type").and_then(Value::as_str) == Some("object") + && !result.contains_key("additionalProperties") + { + result.insert("additionalProperties".into(), json!(false)); + } + for key in ["properties", "$defs", "definitions"] { + if let Some(Value::Object(children)) = result.get(key).cloned() { + let closed: Map = children + .iter() + .map(|(name, child)| (name.clone(), with_closed_objects(child))) + .collect(); + result.insert(key.into(), Value::Object(closed)); + } + } + if let Some(items @ Value::Object(_)) = result.get("items").cloned() { + result.insert("items".into(), with_closed_objects(&items)); + } + for key in ["anyOf", "allOf", "oneOf"] { + if let Some(Value::Array(items)) = result.get(key).cloned() { + result.insert( + key.into(), + Value::Array(items.iter().map(with_closed_objects).collect()), + ); + } + } + Value::Object(result) +} + +/// `_create_output_config_for_response_format`. +pub fn output_config_for_response_format( + schema: Option<&Value>, + name: Option<&Value>, + description: Option<&Value>, +) -> Value { + let schema_text = match schema { + Some(schema) => python_dumps(&with_closed_objects(schema)), + None => "{}".to_owned(), + }; + let mut json_schema = Map::from_iter([("schema".to_owned(), json!(schema_text))]); + if let Some(name) = name.filter(|v| !v.is_null()) { + json_schema.insert("name".into(), name.clone()); + } + if let Some(description) = description.filter(|v| !v.is_null()) { + json_schema.insert("description".into(), description.clone()); + } + json!({"textFormat": {"type": "json_schema", "structure": {"jsonSchema": json_schema}}}) +} + +/// What `map_openai_params` hands to the request build. +#[derive(Debug, Default)] +pub struct MappedParams { + pub params: Map, + pub json_mode: bool, + /// The schema of the `json_tool_call` tool a `response_format` added. + pub response_format_schema: Option, +} + +fn add_tools(out: &mut Map, tools: Vec) { + match out.get_mut("tools") { + Some(Value::Array(existing)) => existing.extend(tools), + _ => { + out.insert("tools".into(), Value::Array(tools)); + } + } +} + +/// `_handle_reasoning_effort_parameter`. +fn map_reasoning_effort_param( + out: &mut Map, + model: &BedrockModel<'_>, + effort: &str, +) -> Result<(), Error> { + if model.model.contains("gpt-oss") || model.model.contains("deepseek") { + out.insert("reasoning_effort".into(), json!(effort)); + } else if is_openai_gpt_reasoning_model(&model.model) { + out.insert("reasoning".into(), json!({"effort": effort})); + } else if is_nova_2_model(&model.model) { + if !matches!(effort, "low" | "medium" | "high") { + return Err(unsupported_value(format!( + "Invalid reasoning_effort value '{effort}' for Nova 2 models. Supported values: ['low', 'medium', 'high']" + ))); + } + out.insert( + "reasoningConfig".into(), + json!({"type": "enabled", "maxReasoningEffort": effort}), + ); + } else { + match map_reasoning_effort(effort, &model.caps)? { + None => { + out.remove("thinking"); + out.remove("output_config"); + } + Some(thinking) => { + out.insert("thinking".into(), thinking); + if model.caps.is_adaptive()? { + let mapped = output_config_effort(effort) + .ok_or_else(|| unmapped_reasoning_effort(effort))?; + let mut output_config = match out.get("output_config") { + Some(Value::Object(existing)) => existing.clone(), + _ => Map::new(), + }; + output_config + .entry("effort") + .or_insert_with(|| json!(mapped)); + model.normalize_output_config_effort(&mut output_config); + if let Some(effort) = output_config.get("effort") { + model.validate_adaptive_effort(effort)?; + } + out.insert("output_config".into(), Value::Object(output_config)); + out.insert("_output_config_normalized".into(), json!(true)); + } + } + } + } + Ok(()) +} + +/// `AmazonConverseConfig.map_openai_params`. +pub fn map_openai_params( + openai: &Map, + model: &BedrockModel<'_>, + drop: DropPolicy, +) -> Result { + let thinking_requested = is_thinking_enabled(openai); + let mut out = Map::new(); + let mut json_mode = false; + let mut response_format_schema = None; + for (name, value) in ordered(openai) { + match name { + "response_format" => { + let Some(format) = value.as_object() else { + continue; + }; + if format.get("type").and_then(Value::as_str) == Some("text") { + continue; + } + let (schema, schema_name, description) = match format.get("response_schema") { + Some(schema) => (Some(schema.clone()), None, None), + None => match format.get("json_schema") { + Some(json_schema) => ( + json_schema.get("schema").cloned(), + json_schema.get("name").cloned(), + json_schema.get("description").cloned(), + ), + None => (None, None, None), + }, + }; + let schema = schema.filter(|schema| !schema.is_null()); + if schema.is_some() && model.supports("supports_native_structured_output")? { + out.insert( + "outputConfig".into(), + output_config_for_response_format( + schema.as_ref(), + schema_name.as_ref(), + description.as_ref(), + ), + ); + } else if let Some(schema) = schema { + let mut function = Map::from_iter([ + ("name".to_owned(), json!(RESPONSE_FORMAT_TOOL_NAME)), + ("parameters".to_owned(), schema.clone()), + ]); + if let Some(description) = + description.filter(|d| d.as_str().is_some_and(|d| !d.is_empty())) + { + function.insert("description".into(), description); + } + add_tools( + &mut out, + vec![json!({"type": "function", "function": function})], + ); + response_format_schema = ResponseFormatSchema::from_schema(&schema); + if model.supports("supports_tool_choice")? + && !thinking_requested + && !model.caps.forced_tool_use_unsupported()? + { + out.insert( + "tool_choice".into(), + json!({"tool": {"name": RESPONSE_FORMAT_TOOL_NAME}}), + ); + } + } + json_mode = true; + } + "max_tokens" | "max_completion_tokens" => { + let requires_min = is_openai_gpt_reasoning_model(&model.model) + || model.model.contains("xai.grok-"); + let value = match value.as_i64() { + Some(tokens) if requires_min && value.is_i64() => { + json!(tokens.max(OPENAI_COMPAT_MIN_MAX_TOKENS)) + } + _ => value.clone(), + }; + out.insert("maxTokens".into(), value); + } + "stream" => { + out.insert("stream".into(), value.clone()); + } + "stop" if !rejects_stop_sequences(&model.model) => { + let stop = match value { + Value::String(text) if text.is_empty() => continue, + Value::String(text) => json!([text]), + other => other.clone(), + }; + out.insert("stopSequences".into(), stop); + } + "temperature" | "top_p" => { + let output_key = match name { + "top_p" => "topP", + _ => name, + }; + if model.is_anthropic() { + apply_sampling_param(&mut out, &model.caps, drop, name, value, output_key)?; + } else if !model.supports_sampling()? { + drop.unsupported(|| { + unsupported_value(format!( + "{} does not support {name}={}. To drop unsupported params, set `litellm.drop_params = True`.", + model.model, + python_str(value) + )) + })?; + } else { + out.insert(output_key.into(), value.clone()); + } + } + "tools" => { + let Some(tools) = value.as_array() else { + continue; + }; + if tools + .iter() + .any(|tool| tool.get("type").and_then(Value::as_str) == Some("tool_search")) + { + return Err(Error::Unsupported("the OpenAI tool_search tool on Bedrock")); + } + add_tools(&mut out, tools.clone()); + } + "tool_choice" => { + if let Some(choice) = map_tool_choice_values(model, value, drop)? { + out.insert("tool_choice".into(), choice); + } + } + "parallel_tool_calls" => { + out.insert( + "_parallel_tool_use_config".into(), + json!({"tool_choice": {"type": "auto", "disable_parallel_tool_use": !crate::base_llm::chat::params::truthy(value)}}), + ); + } + "thinking" if model.is_deepseek() => {} + "thinking" if !is_openai_gpt_reasoning_model(&model.model) => { + let adaptive_request = + value.get("type").and_then(Value::as_str) == Some("adaptive"); + if adaptive_request && !model.caps.is_adaptive()? { + let max_tokens = openai + .get("max_completion_tokens") + .filter(|v| crate::base_llm::chat::params::truthy(v)) + .or_else(|| openai.get("max_tokens")) + .filter(|v| !v.is_null()); + let legacy = map_reasoning_effort("medium", &model.caps)?; + if let Some(capped) = + legacy.and_then(|legacy| cap_thinking_budget(legacy, max_tokens)) + { + out.insert("thinking".into(), capped); + } + } else { + out.insert("thinking".into(), value.clone()); + translate_legacy_thinking(&mut out, &model.caps, drop)?; + } + } + "reasoning_effort" => { + let Some(effort) = value.as_str() else { + continue; + }; + if model.is_deepseek_r1() { + continue; + } + if is_openai_gpt_reasoning_model(&model.model) { + return Err(Error::Unsupported( + "reasoning_effort for an OpenAI GPT model on Bedrock Converse", + )); + } + map_reasoning_effort_param(&mut out, model, effort)?; + } + "context_management" => { + if let Some(mapped) = match value { + Value::Array(_) => map_context_management(value), + Value::Object(_) => Some(value.clone()), + _ => None, + } { + out.insert("context_management".into(), mapped); + } + } + "service_tier" => { + let Some(tier) = value.as_str() else { continue }; + let tier = match tier { + "auto" => "default", + other => other, + }; + if matches!(tier, "default" | "flex" | "priority") { + out.insert("serviceTier".into(), json!({"type": tier})); + } + } + "web_search_options" + if value.is_object() && model.model.to_lowercase().contains("nova") => + { + add_tools( + &mut out, + vec![json!({"systemTool": {"name": "nova_grounding"}})], + ); + } + _ => {} + } + } + if !model.model.contains("gpt-oss") && !is_nova_2_model(&model.model) { + if let Some(Value::Object(thinking)) = out.get_mut("thinking") + && let Some(budget) = thinking.get("budget_tokens").and_then(Value::as_i64) + && thinking.get("budget_tokens").is_some_and(Value::is_i64) + && budget < MIN_THINKING_BUDGET_TOKENS + { + thinking.insert("budget_tokens".into(), json!(MIN_THINKING_BUDGET_TOKENS)); + } + let max_tokens_in_request = + openai.contains_key("max_tokens") || openai.contains_key("max_completion_tokens"); + if is_thinking_enabled(&out) + && !max_tokens_in_request + && let Some(budget) = out + .get("thinking") + .and_then(|thinking| thinking.get("budget_tokens")) + .and_then(Value::as_i64) + { + out.insert( + "maxTokens".into(), + json!(budget + DEFAULT_MAX_TOKENS as i64), + ); + } + } + if is_thinking_enabled(&out) + && out + .get("tool_choice") + .and_then(Value::as_object) + .is_some_and(|choice| choice.contains_key("any") || choice.contains_key("tool")) + { + out.insert("tool_choice".into(), json!({"auto": {}})); + } + Ok(MappedParams { + params: out, + json_mode, + response_format_schema, + }) +} + +/// `_validate_request_metadata`. +pub fn validate_request_metadata(metadata: &Value) -> Result<(), Error> { + let Some(metadata) = metadata.as_object() else { + return Err(unsupported_value("requestMetadata must be a dictionary")); + }; + if metadata.len() > 16 { + return Err(unsupported_value( + "requestMetadata can contain a maximum of 16 items", + )); + } + let allowed = + |c: char| c.is_ascii_alphanumeric() || c.is_whitespace() || ":_@$#=/+,.-".contains(c); + for (key, value) in metadata { + let Some(value) = value.as_str() else { + return Err(unsupported_value("requestMetadata values must be strings")); + }; + if key.is_empty() || key.chars().count() > 256 { + return Err(unsupported_value( + "requestMetadata key length must be 1-256 characters", + )); + } + if value.chars().count() > 256 { + return Err(unsupported_value( + "requestMetadata value length must be 0-256 characters", + )); + } + if !key.chars().all(allowed) { + return Err(unsupported_value(format!( + "requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]" + ))); + } + if !value.chars().all(allowed) { + return Err(unsupported_value(format!( + "requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]" + ))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case( + "us.meta.llama3-2-11b-instruct-v1:0", + "meta.llama3-2-11b-instruct-v1:0" + )] + #[case( + "bedrock/converse/anthropic.claude-3-5-sonnet-20240620-v1:0", + "anthropic.claude-3-5-sonnet-20240620-v1:0" + )] + #[case( + "anthropic.claude-3-5-sonnet-20241022-v2:0:51k", + "anthropic.claude-3-5-sonnet-20241022-v2:0" + )] + #[case("bedrock/nova-2/arn:aws:bedrock:x", "amazon.nova-2-custom")] + #[case( + "arn:aws:bedrock:us-east-1:1:inference-profile/us.anthropic.claude-x", + "anthropic.claude-x" + )] + #[case("us-west-2/mistral.mistral-large", "mistral.mistral-large")] + #[case("global.anthropic.claude-sonnet-4-5", "anthropic.claude-sonnet-4-5")] + fn base_models_match_python(#[case] model: &str, #[case] expected: &str) { + assert_eq!(bedrock_base_model(model), expected); + } + + #[rstest] + #[case("get weather", "get_weather")] + #[case("1tool", "a1tool")] + #[case("", "")] + #[case("ok-name_1", "ok-name_1")] + fn tool_names_are_made_valid(#[case] name: &str, #[case] expected: &str) { + assert_eq!(make_valid_bedrock_tool_name(name), expected); + } + + #[rstest] + fn gpt_versions_parse() { + assert_eq!(openai_gpt_version("openai.gpt-5.6"), Some((5, 6))); + assert!(rejects_stop_sequences("us.xai.grok-4")); + assert!(!rejects_stop_sequences("openai.gpt-oss-20b-1:0")); + } + + #[rstest] + fn request_metadata_is_validated() { + assert!(validate_request_metadata(&json!({"a": "b"})).is_ok()); + assert_eq!( + validate_request_metadata(&json!({"a": 1})) + .unwrap_err() + .to_string(), + "invalid request: requestMetadata values must be strings" + ); + } +} diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs index a27a71b2802..5bcfb02e730 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -1,3 +1,6 @@ +//! Python's `AmazonConverseConfig` for `/chat/completions`: `get_optional_params` and +//! `_transform_request` on the way in, `_transform_response` on the way out. + use litellm_auth::AwsParams; use litellm_auth::{CredentialPlacement, SecretValue}; use litellm_auth_aws::{ @@ -5,51 +8,51 @@ use litellm_auth_aws::{ constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, resolve_bedrock_region, }; -use litellm_core_utils::{ - core_helpers::{finish_reason_for, unix_now, usage_from_parts}, - prompt_templates::factory::{Conversation, TurnRole, build_conversation}, -}; +use litellm_core_utils::core_helpers::{finish_reason_for, unix_now, usage_from_parts}; use litellm_llms_types::{ formats::chat_completions::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, - ChatCompletionsUsage, ChatMessage, ChatMessageContent, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, ChatCompletionsChoice, ChatCompletionsChoiceMessage, + ChatCompletionsResponse, ChatCompletionsUsage, ChatMessage, }, - providers::bedrock::{CONVERSE_PATH, ConverseResponse}, + providers::bedrock::{CONVERSE_PATH, ConverseUsage}, }; +use serde::Deserialize; use serde_json::{Map, Value, json}; use crate::{ - Error, + Error, ErrorDetail, + anthropic::chat::{messages::native_redacted_thinking_data, params::apply_sampling_param}, base_llm::{ auth::AuthScheme, chat::{ + media::{data_url, remote_urls}, + params::{ + ChatRequestContext, RESPONSE_FORMAT_TOOL_NAME, python_dumps, sort_params, truthy, + }, streaming::{ChatStream, StreamShape}, transformation::{ BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, - Unsupported, ValidatedEnvironment, unsupported_message, unsupported_param, + Unsupported, ValidatedEnvironment, }, }, }, - bedrock::chat::invoke_handler::converse_chat_stream, + bedrock::chat::{ + converse_messages::{ + converse_messages, converse_tools, guard_trailing_user_messages, + neutralize_orphaned_tool_blocks, transform_system_message, + }, + converse_params::{ + BedrockModel, MappedParams, map_openai_params, output_config_for_response_format, + supported_params, validate_request_metadata, + }, + invoke_handler::converse_chat_stream, + }, }; /// The action a streaming Converse call goes to, in place of [`CONVERSE_PATH`]. const CONVERSE_STREAM_PATH: &str = "/converse-stream"; -/// Converse parameter names, post `map_openai_params`, that the Rust path can -/// place verbatim in `inferenceConfig`. -/// -/// `topK` is deliberately absent: Python routes it to -/// `additionalModelRequestFields` for Anthropic base models and to -/// `inferenceConfig` otherwise, and that branch reads the model catalog the -/// core cannot see. -const SUPPORTED_PARAMS: &[(&str, &str)] = &[ - ("max_tokens", "maxTokens"), - ("temperature", "temperature"), - ("top_p", "topP"), - ("stop", "stopSequences"), -]; - const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "aws_bedrock_runtime_endpoint"; /// AWS call configuration a host passes down: consumed for signing and endpoint @@ -68,6 +71,30 @@ const CONFIG_PARAMS: &[&str] = &[ AWS_BEDROCK_RUNTIME_ENDPOINT, ]; +/// `AmazonConverseConfig.__annotations__` plus `top_k`, the tool params and the config +/// blocks: what stays out of `additionalModelRequestFields`. +const INFERENCE_PARAMS: [&str; 6] = [ + "maxTokens", + "stopSequences", + "temperature", + "topP", + "topK", + "top_k", +]; +const TOOL_PARAMS: [&str; 2] = ["tools", "tool_choice"]; +const CONFIG_BLOCKS: [&str; 3] = ["guardrailConfig", "performanceConfig", "serviceTier"]; +/// `filter_internal_params`. +const INTERNAL_PARAMS: [&str; 3] = [ + "skip_mcp_handler", + "mcp_handler_context", + "_skip_mcp_handler", +]; + +const FINE_GRAINED_TOOL_STREAMING_BETA: &str = "fine-grained-tool-streaming-2025-05-14"; +const EFFORT_BETA: &str = "effort-2025-11-24"; +const COMPACT_BETA: &str = "compact-2026-01-12"; +const COMPACT_EDIT_TYPE: &str = "compact_20260112"; + enum ConverseStopReason { Value(String), Unknown(String), @@ -92,10 +119,552 @@ impl ConverseStopReason { } } +#[derive(Deserialize)] +struct ResponseProjection { + output: OutputProjection, + #[serde(default)] + usage: ConverseUsage, + #[serde(rename = "stopReason")] + stop_reason: Option, +} + +#[derive(Deserialize)] +struct OutputProjection { + message: Option, +} + +#[derive(Deserialize)] +struct MessageProjection { + content: Vec, +} + +#[derive(Deserialize)] +struct ContentProjection { + #[serde(default)] + text: Option, + #[serde(default, rename = "toolUse")] + tool_use: Option, + #[serde(default, rename = "reasoningContent")] + reasoning_content: Option, + #[serde(default, rename = "citationsContent")] + citations_content: Option, +} + pub struct AmazonConverseConfig; pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: AmazonConverseConfig = AmazonConverseConfig; +/// `parse_content_for_reasoning`: a leading `` block split from the answer. +fn parse_content_for_reasoning(text: &str) -> (Option, String) { + for tag in ["think", "thinking", "budget:thinking"] { + let Some(rest) = text.strip_prefix(&format!("<{tag}>")) else { + continue; + }; + let closes = ["think", "thinking", "budget:thinking"] + .iter() + .filter_map(|close| { + let marker = format!(""); + rest.find(&marker).map(|at| (at, marker.len())) + }) + .min_by_key(|(at, _)| *at); + if let Some((at, len)) = closes { + return (Some(rest[..at].to_owned()), rest[at + len..].to_owned()); + } + } + (None, text.to_owned()) +} + +/// `_unwrap_bedrock_properties`: a lone `properties` key gives up its value. +pub fn unwrap_bedrock_properties(text: &str) -> String { + match serde_json::from_str::(text) { + Ok(Value::Object(object)) if object.len() == 1 && object.contains_key("properties") => { + python_dumps(&object["properties"]) + } + _ => text.to_owned(), + } +} + +/// `eager_input_streaming_flag` over a chat tool. +fn eager_input_streaming(tool: &Value) -> bool { + tool.get("eager_input_streaming") == Some(&Value::Bool(true)) + || tool + .get("function") + .and_then(|function| function.get("eager_input_streaming")) + == Some(&Value::Bool(true)) +} + +/// `_merge_parallel_tool_use_config`. +fn merge_parallel_tool_use_config(additional: &mut Map, config: Map) { + for (key, value) in config { + let merged = match (additional.get(&key), &value) { + (Some(Value::Object(existing)), Value::Object(incoming)) => { + let mut merged = incoming.clone(); + merged.extend(existing.iter().map(|(k, v)| (k.clone(), v.clone()))); + merged.extend( + incoming + .iter() + .filter(|(k, _)| k.as_str() != "type") + .map(|(k, v)| (k.clone(), v.clone())), + ); + Value::Object(merged) + } + _ => value, + }; + additional.insert(key, merged); + } +} + +impl AmazonConverseConfig { + fn build_request( + &self, + model: &str, + messages: Vec, + optional_params: Map, + context: &ChatRequestContext<'_>, + ) -> Result { + let bedrock = BedrockModel::new(model, context); + let sorted = sort_params( + optional_params, + CONFIG_PARAMS, + context, + |_| supported_params(&bedrock), + "bedrock", + model, + )?; + let drop = sorted.drop; + let MappedParams { + mut params, + json_mode, + response_format_schema, + } = map_openai_params(&sorted.openai, &bedrock, drop)?; + for name in &sorted.allowed { + if let Some(value) = sorted.openai.get(name) + && !params.contains_key(name) + { + params.insert(name.clone(), value.clone()); + } + } + params.extend(sorted.provider); + + let messages: Vec = messages + .iter() + .map(|message| { + serde_json::to_value(message).map(|mut value| { + if value.get("role").and_then(Value::as_str) == Some("developer") { + value["role"] = json!("system"); + } + value + }) + }) + .collect::>() + .map_err(|error| Error::InvalidRequest(ErrorDetail::invalid("messages", error)))?; + let (messages, system) = transform_system_message(messages, &bedrock); + let guardrail = params.contains_key("guardrailConfig"); + let has_tools = params.get("tools").is_some_and(truthy); + let mut messages = neutralize_orphaned_tool_blocks(messages, has_tools, guardrail); + if guardrail { + messages = guard_trailing_user_messages(messages); + } + + // `maybe_drop_disabled_thinking`. + if params + .get("thinking") + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("disabled") + && bedrock.caps.supports("thinking_always_on")? + { + params.remove("thinking"); + } + + // `_prepare_request_params`. + let normalized = params + .remove("_output_config_normalized") + .is_some_and(|flag| truthy(&flag)); + for internal in ["json_mode", "stream", "fake_stream"] { + params.remove(internal); + } + let mut anthropic_output_config = match params.remove("output_config") { + Some(Value::Object(config)) => Some(config), + _ => None, + }; + let mut output_config_format = None; + if let Some(config) = &mut anthropic_output_config { + if let Some(Value::Object(format)) = config.remove("format") { + output_config_format = Some(format); + } + if config.is_empty() { + anthropic_output_config = None; + } + } + let request_metadata = params.remove("requestMetadata").filter(|v| !v.is_null()); + if let Some(metadata) = &request_metadata { + validate_request_metadata(metadata)?; + } + let mut output_config = params.remove("outputConfig").filter(|v| !v.is_null()); + if output_config.is_none() + && let Some(format) = &output_config_format + && format.get("type").and_then(Value::as_str) == Some("json_schema") + && bedrock.is_anthropic() + && bedrock.supports("supports_native_structured_output")? + { + output_config = Some(output_config_for_response_format( + format.get("schema"), + format.get("name"), + format.get("description"), + )); + } + let (mut inference, mut additional): (Map, Map) = + params.into_iter().partition(|(key, _)| { + INFERENCE_PARAMS.contains(&key.as_str()) + || TOOL_PARAMS.contains(&key.as_str()) + || CONFIG_BLOCKS.contains(&key.as_str()) + }); + if let Some(Value::Object(config)) = additional.remove("_parallel_tool_use_config") + && bedrock.supports_parallel_tool_use_config()? + { + merge_parallel_tool_use_config(&mut additional, config); + } + additional.remove("parallel_tool_calls"); + additional.remove("client_metadata"); + let top_k = inference + .remove("topK") + .or_else(|| inference.remove("top_k")) + .filter(|top_k| !top_k.is_null()); + if let Some(top_k) = top_k { + if bedrock.is_anthropic() { + apply_sampling_param( + &mut additional, + &bedrock.caps, + drop, + "top_k", + &top_k, + "top_k", + )?; + } else if bedrock.base.starts_with("amazon.nova") { + additional.insert("inferenceConfig".into(), json!({"topK": top_k})); + } + } + for internal in INTERNAL_PARAMS { + additional.remove(internal); + } + if let Some(mut config) = anthropic_output_config { + if model.contains(":application-inference-profile/") { + additional.insert("output_config".into(), Value::Object(config)); + } else if bedrock.is_anthropic() + && !(drop.drops() && !bedrock.supports_effort_param()?) + { + if !normalized { + bedrock.normalize_output_config_effort(&mut config); + } + if let Some(effort) = config.get("effort").filter(|effort| !effort.is_null()) { + bedrock.validate_adaptive_effort(effort)?; + } + additional.insert("output_config".into(), Value::Object(config)); + } + } + + // `_process_tools_and_beta`. + let tools = match inference.remove("tools") { + Some(Value::Array(tools)) => tools, + _ => Vec::new(), + }; + let (tool_blocks, tool_name_reverse_map) = converse_tools(&tools, &bedrock)?; + let mut betas: Vec<&str> = Vec::new(); + if bedrock.is_anthropic() { + if tools.iter().any(eager_input_streaming) { + betas.push(FINE_GRAINED_TOOL_STREAMING_BETA); + } + if additional + .get("output_config") + .and_then(|config| config.get("effort")) + .is_some_and(|effort| !effort.is_null()) + && !bedrock.caps.is_adaptive()? + { + betas.push(EFFORT_BETA); + } + let compact_edits: Option<(Map, Vec)> = match additional + .get("context_management") + { + Some(Value::Object(management)) => match management.get("edits") { + Some(Value::Array(edits)) => Some(( + management.clone(), + edits + .iter() + .filter(|edit| { + edit.get("type").and_then(Value::as_str) == Some(COMPACT_EDIT_TYPE) + }) + .cloned() + .collect(), + )), + _ => None, + }, + _ => None, + }; + match compact_edits { + Some((mut management, edits)) if !edits.is_empty() => { + betas.push(COMPACT_BETA); + management.insert("edits".into(), Value::Array(edits)); + additional.insert("context_management".into(), Value::Object(management)); + } + _ => { + additional.remove("context_management"); + } + } + if !betas.is_empty() { + additional.insert("anthropic_beta".into(), json!(betas)); + } + } + additional.remove("cache_control_injection_points"); + + let mut tool_config = None; + if !tool_blocks.is_empty() { + let mut config = Map::from_iter([("tools".to_owned(), Value::Array(tool_blocks))]); + if let Some(choice) = inference.remove("tool_choice") { + config.insert("toolChoice".into(), choice); + if let Some(Value::Object(choice)) = additional.get_mut("tool_choice") { + choice.remove("type"); + } + } + tool_config = Some(Value::Object(config)); + } + let config_blocks: Vec<(&str, Option)> = CONFIG_BLOCKS + .iter() + .map(|name| (*name, inference.remove(*name))) + .collect(); + + let mut body = Map::from_iter([( + "messages".to_owned(), + Value::Array(converse_messages(&messages, &bedrock)?), + )]); + body.insert("inferenceConfig".into(), Value::Object(inference)); + if !additional.is_empty() { + let thinking = additional.contains_key("thinking"); + body.insert( + "additionalModelRequestFields".into(), + Value::Object(additional), + ); + if thinking { + body.insert( + "additionalModelResponseFieldPaths".into(), + json!(["/usage/output_tokens_details"]), + ); + } + } + if !system.is_empty() { + body.insert("system".into(), Value::Array(system)); + } + for (name, value) in config_blocks { + if let Some(value) = value.filter(|value| !value.is_null()) { + body.insert(name.into(), value); + } + } + if let Some(tool_config) = tool_config { + body.insert("toolConfig".into(), tool_config); + } + if let Some(metadata) = request_metadata { + body.insert("requestMetadata".into(), metadata); + } + if let Some(output_config) = output_config { + body.insert("outputConfig".into(), output_config); + } + Ok(ProviderChatRequestData { + body: Value::Object(body), + stream_shape: StreamShape { + json_mode, + tool_name_reverse_map, + response_format_schema, + ..StreamShape::default() + }, + headers: Vec::new(), + }) + } +} + +/// `_transform_thinking_blocks`. +fn thinking_blocks(blocks: &[Value]) -> Vec { + blocks + .iter() + .filter_map(|block| { + if let Some(text) = block.get("reasoningText") { + return Some(ChatCompletionThinkingBlock::Thinking { + thinking: text.get("text").and_then(Value::as_str).map(str::to_owned), + signature: text + .get("signature") + .and_then(Value::as_str) + .map(str::to_owned), + cache_control: None, + index: None, + }); + } + block + .get("redactedContent") + .and_then(Value::as_str) + .map(|data| ChatCompletionThinkingBlock::RedactedThinking { + data: Some(native_redacted_thinking_data(data)), + cache_control: None, + index: None, + }) + }) + .collect() +} + +fn converse_response( + model: &str, + response: ResponseProjection, + shape: &StreamShape, +) -> ChatCompletionsResponse { + let mut content = String::new(); + let mut tool_calls: Vec = Vec::new(); + let mut reasoning_blocks: Option> = None; + let mut citations: Option> = None; + for (index, block) in response + .output + .message + .map(|message| message.content) + .unwrap_or_default() + .into_iter() + .enumerate() + { + let mut extracted_reasoning = None; + if let Some(text) = &block.text { + let (reasoning, answer) = parse_content_for_reasoning(text); + extracted_reasoning = reasoning; + content.push_str(&answer); + } + if let Some(tool) = &block.tool_use { + let name = tool.get("name").and_then(Value::as_str).unwrap_or_default(); + tool_calls.push(ChatCompletionToolCallChunk { + id: tool + .get("toolUseId") + .and_then(Value::as_str) + .map(str::to_owned), + tool_type: "function".into(), + function: ChatCompletionToolCallFunctionChunk { + name: Some( + shape + .tool_name_reverse_map + .get(name) + .cloned() + .unwrap_or_else(|| name.to_owned()), + ), + arguments: python_dumps(tool.get("input").unwrap_or(&Value::Null)), + provider_specific_fields: None, + }, + index: index as i64, + }); + } + if let Some(reasoning) = extracted_reasoning { + reasoning_blocks + .get_or_insert_with(Vec::new) + .push(json!({"reasoningText": {"text": reasoning}})); + } + if let Some(reasoning) = block.reasoning_content { + reasoning_blocks + .get_or_insert_with(Vec::new) + .push(reasoning); + } + if let Some(citation) = block.citations_content { + citations.get_or_insert_with(Vec::new).push(citation); + } + } + let mut provider_fields = Map::new(); + if let Some(blocks) = &reasoning_blocks { + provider_fields.insert( + "reasoningContentBlocks".into(), + Value::Array(blocks.clone()), + ); + } + if let Some(citations) = citations { + provider_fields.insert("citationsContent".into(), Value::Array(citations)); + } + let reasoning_content = reasoning_blocks.as_ref().map(|blocks| { + blocks + .iter() + .filter_map(|block| block.get("reasoningText")?.get("text")?.as_str()) + .collect::() + }); + let thinking = reasoning_blocks.as_deref().map(thinking_blocks); + + // `_filter_json_mode_tools`. + let had_tools = !tool_calls.is_empty(); + let json_indices: Vec = match shape.json_mode { + true => tool_calls + .iter() + .enumerate() + .filter(|(_, call)| call.function.name.as_deref() == Some(RESPONSE_FORMAT_TOOL_NAME)) + .map(|(index, _)| index) + .collect(), + false => Vec::new(), + }; + let filtered: Vec = if json_indices.is_empty() { + tool_calls + } else { + let answer = unwrap_bedrock_properties(&tool_calls[json_indices[0]].function.arguments); + if json_indices.len() == tool_calls.len() { + content = answer; + Vec::new() + } else { + content = match content.is_empty() { + true => answer, + false => format!("{content}{answer}"), + }; + tool_calls + .into_iter() + .enumerate() + .filter(|(index, _)| !json_indices.contains(index)) + .map(|(_, call)| call) + .collect() + } + }; + let mapped = finish_reason_for( + ConverseStopReason::parse(response.stop_reason) + .as_ref() + .map(ConverseStopReason::as_str) + .unwrap_or(""), + ) + .to_string(); + let finish_reason = + match shape.json_mode && filtered.is_empty() && had_tools && mapped == "tool_calls" { + true => "stop".to_owned(), + false => mapped, + }; + + let computed = usage_from_parts( + response.usage.input_tokens, + response.usage.output_tokens, + response.usage.cache_read_input_tokens, + response.usage.cache_write_input_tokens, + ); + // Converse reports `totalTokens` and Python passes it straight through. Python does a + // bare `usage["totalTokens"]` lookup, so a body without the key raises there; the + // computed total stands in rather than failing the call. + let usage = ChatCompletionsUsage { + total_tokens: response.usage.total_tokens.unwrap_or(computed.total_tokens), + ..computed + }; + ChatCompletionsResponse { + created: unix_now(), + // Converse echoes no model id, so Python reports the requested one. + model: model.to_string(), + choices: vec![ChatCompletionsChoice { + index: 0, + message: ChatCompletionsChoiceMessage { + role: "assistant".to_string(), + // Converse assigns the joined string unconditionally, so an empty answer + // is `""` here and not `None` as it is on Anthropic. + content: Some(content), + tool_calls: (!filtered.is_empty()).then_some(filtered), + reasoning_content, + thinking_blocks: thinking, + provider_specific_fields: (!provider_fields.is_empty()).then_some(provider_fields), + }, + finish_reason, + }], + usage, + } +} + impl BaseConfig for AmazonConverseConfig { fn secret_names(&self) -> Vec<&'static str> { litellm_auth::AwsParams::secret_names() @@ -105,10 +674,6 @@ impl BaseConfig for AmazonConverseConfig { .collect() } - fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)] { - SUPPORTED_PARAMS - } - fn get_complete_url( &self, api_base: Option<&str>, @@ -142,20 +707,19 @@ impl BaseConfig for AmazonConverseConfig { fn transform_request( &self, - _model: &str, + model: &str, messages: Vec, optional_params: Map, + context: &ChatRequestContext<'_>, ) -> Result { - Ok(ProviderChatRequestData { - body: converse_body(&build_conversation(&messages), &optional_params), - stream_shape: StreamShape::default(), - }) + self.build_request(model, messages, optional_params, context) } fn transform_response( &self, model: &str, response: ProviderChatResponseData, + shape: &StreamShape, ) -> Result { let body = response.body; if !body.is_object() { @@ -170,55 +734,18 @@ impl BaseConfig for AmazonConverseConfig { )); } } - let response: ConverseResponse = serde_json::from_value(body).map_err(|error| { + let response: ResponseProjection = serde_json::from_value(body).map_err(|error| { Error::InvalidResponse(format!("invalid Converse response: {error}").into()) })?; - // The route declines tool requests, so anything other than a text block - // is something this path never asked for. Decline; the host falls back. - if response.message_content_is_non_text() { - return Err(Error::Unsupported("non-text response content block")); - } - let text = response.content_text(); - let computed = usage_from_parts( - response.usage.input_tokens, - response.usage.output_tokens, - response.usage.cache_read_input_tokens, - response.usage.cache_write_input_tokens, - ); - // Converse reports `totalTokens` and Python passes it straight through, - // where Anthropic has no such field and Python adds the two counts - // instead, so only this provider overrides the computed total. Python - // does a bare `usage["totalTokens"]` lookup, so a body without the key - // raises there rather than reporting a zero; fall back to the computed - // total, which is the closest thing to that without failing the call. - let usage = ChatCompletionsUsage { - total_tokens: response.usage.total_tokens.unwrap_or(computed.total_tokens), - ..computed - }; + Ok(converse_response(model, response, shape)) + } - Ok(ChatCompletionsResponse { - created: unix_now(), - // Converse echoes no model id, so Python reports the requested one. - model: model.to_string(), - choices: vec![ChatCompletionsChoice { - index: 0, - message: ChatCompletionsChoiceMessage { - role: "assistant".to_string(), - // Converse assigns the joined string unconditionally, so an - // empty response is `""` here and not `None` as it is on - // Anthropic. A caller calling `.strip()` on it would break - // on this path alone. - content: Some(text), - }, - finish_reason: finish_reason_for( - ConverseStopReason::parse(response.stop_reason) - .as_ref() - .map(ConverseStopReason::as_str) - .unwrap_or(""), - ) - .to_string(), - }], - usage, + /// Python's `BedrockImageProcessor` downloads every image or file URL a message names. + fn remote_media(&self, messages: &[ChatMessage]) -> Vec { + remote_urls(messages, &["user", "assistant", "tool"], true, |url| { + data_url(url).is_none() + && !url.contains("base64") + && (url.contains("http://") || url.contains("https://")) }) } @@ -282,100 +809,30 @@ impl BaseConfig for AmazonConverseConfig { CONFIG_PARAMS } + /// Every request is translated; a param or message Python refuses is refused here too. fn unsupported_reason( &self, - messages: &[ChatMessage], - optional_params: &Map, + _messages: &[ChatMessage], + _optional_params: &Map, ) -> Option { - unsupported_param( - self.supported_openai_param_mappings(), - CONFIG_PARAMS, - optional_params, - ) - .or_else(|| messages.iter().find_map(unsupported_message)) - // Python's Converse translation drops blank text blocks instead of - // substituting the placeholder the shared conversation builder - // applies, so decline blank text rather than diverge. - .or_else(|| { - messages - .iter() - .any(has_blank_text) - .then_some(Unsupported("blank message text")) - }) - // Converse has no assistant prefill: Python inserts a continue turn - // when a conversation opens or closes on an assistant message, and - // only under `litellm.modify_params`, which the core cannot see. - // Declining both ends also keeps the shared builder's final - // assistant right-strip (an Anthropic rule) unreachable here. - .or_else(|| { - let conversation = build_conversation(messages); - let ends_on_assistant = conversation - .turns - .last() - .is_some_and(|turn| turn.role == TurnRole::Assistant); - (!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Unsupported( - "conversation does not run user turn to user turn", - )) - }) - } -} - -fn converse_body(conversation: &Conversation, optional_params: &Map) -> Value { - let messages: Vec = conversation - .turns - .iter() - .map(|turn| { - json!({ - "role": <&'static str>::from(turn.role), - "content": turn.texts.iter().map(|text| json!({"text": text})).collect::>(), - }) - }) - .collect(); - - let inference_config = Map::from_iter(SUPPORTED_PARAMS.iter().filter_map(|(_, name)| { - optional_params - .get(*name) - .map(|value| ((*name).to_string(), value.clone())) - })); - - let system: Vec = conversation - .system - .iter() - .map(|text| json!({"text": text})) - .collect(); - - Value::Object(Map::from_iter( - [ - ( - "inferenceConfig".to_string(), - Value::Object(inference_config), - ), - ("messages".to_string(), json!(messages)), - ] - .into_iter() - .chain((!system.is_empty()).then(|| ("system".to_string(), json!(system)))), - )) -} - -fn has_blank_text(message: &ChatMessage) -> bool { - match &message.content { - None => false, - Some(ChatMessageContent::Text(text)) => text.trim().is_empty(), - Some(ChatMessageContent::Parts(parts)) => parts.iter().any(|part| { - part.get("text") - .and_then(Value::as_str) - .is_none_or(|text| text.trim().is_empty()) - }), + None } } #[cfg(test)] mod tests { + use std::collections::HashMap as StdHashMap; + use crate::{ Error, base_llm::{ auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, + chat::{ + params::ChatRequestContext, + streaming::StreamShape, + transformation::{BaseConfig, ProviderChatResponseData}, + }, + model_info::ModelInfo, }, bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, }; @@ -384,6 +841,27 @@ mod tests { use rstest::rstest; use serde_json::{Map, Value, json}; + struct CostMap(StdHashMap>); + + impl ModelInfo for CostMap { + fn entry(&self, model: &str) -> Option>> { + self.0.get(model).map(std::borrow::Cow::Borrowed) + } + fn declares(&self, key: &str) -> bool { + self.0.values().any(|entry| entry.contains_key(key)) + } + } + + fn cost_map() -> CostMap { + CostMap(StdHashMap::from([( + "anthropic.claude-sonnet-4-5-v1:0".to_owned(), + json!({"supports_tool_choice": true, "supports_reasoning": true, "supports_prompt_caching": true}) + .as_object() + .cloned() + .unwrap(), + )])) + } + fn messages(value: Value) -> Vec { serde_json::from_value(value).expect("valid messages") } @@ -395,207 +873,204 @@ mod tests { } } - fn transform(msgs: Value, opts: Value) -> Value { - BEDROCK_CHAT_COMPLETIONS_CONFIG - .transform_request( - "anthropic.claude-sonnet-4-5-v1:0", - messages(msgs), - params(opts), - ) - .expect("request transforms") - .body - } - - fn transform_response(body: Value) -> Result { - BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( + fn request( + msgs: Value, + opts: Value, + drop: bool, + ) -> Result { + let info = cost_map(); + BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_request( "anthropic.claude-sonnet-4-5-v1:0", - ProviderChatResponseData { body }, + messages(msgs), + params(opts), + &ChatRequestContext { + drop_params: Some(drop), + model_info: Some(&info), + }, ) } - fn reason(msgs: Value, opts: Value) -> Option { - BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + fn transform(msgs: Value, opts: Value) -> Value { + request(msgs, opts, false).expect("request transforms").body } - #[test] + fn transform_response(body: Value) -> Result { + transform_response_with(body, &StreamShape::default()) + } + + fn transform_response_with( + body: Value, + shape: &StreamShape, + ) -> Result { + BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( + "anthropic.claude-sonnet-4-5-v1:0", + ProviderChatResponseData { body }, + shape, + ) + } + + #[rstest] fn builds_the_converse_body_python_builds() { let body = transform( json!([ {"role": "system", "content": "be terse"}, {"role": "user", "content": "hi"} ]), - json!({"maxTokens": 128, "temperature": 0.2}), + json!({"max_tokens": 128, "temperature": 0.2, "top_p": 0.9, "stop": "END"}), ); assert_eq!( body, json!({ - "inferenceConfig": {"maxTokens": 128, "temperature": 0.2}, "messages": [{"role": "user", "content": [{"text": "hi"}]}], + "inferenceConfig": {"maxTokens": 128, "temperature": 0.2, "topP": 0.9, "stopSequences": ["END"]}, "system": [{"text": "be terse"}] }) ); } - #[test] - fn always_emits_inference_config_even_when_empty() { - let body = transform(json!([{"role": "user", "content": "hi"}]), json!({})); - assert_eq!(body["inferenceConfig"], json!({})); - assert!(body.get("system").is_none()); - } - - #[test] - fn places_only_inference_params_in_inference_config() { + #[rstest] + fn tools_and_tool_choice_become_a_tool_config() { let body = transform( - json!([{"role": "user", "content": "hi"}]), + json!([{"role": "user", "content": "weather?"}]), json!({ - "maxTokens": 64, - "temperature": 0.1, - "topP": 0.9, - "stopSequences": ["STOP"] + "max_tokens": 10, + "tools": [{"type": "function", "function": {"name": "get_weather", "description": "d", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}], + "tool_choice": "required" }), ); assert_eq!( - body["inferenceConfig"], - json!({"maxTokens": 64, "temperature": 0.1, "topP": 0.9, "stopSequences": ["STOP"]}) + body["toolConfig"], + json!({"tools": [{"toolSpec": {"inputSchema": {"json": {"type": "object", "properties": {"city": {"type": "string"}}, "required": []}}, "name": "get_weather", "description": "d"}}], + "toolChoice": {"any": {}}}) ); - assert!(body.get("additionalModelRequestFields").is_none()); } - #[test] - fn merges_consecutive_user_turns_into_one_message() { + #[rstest] + fn tool_choice_none_is_refused_or_dropped() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let error = request(msgs.clone(), json!({"tool_choice": "none"}), false) + .map(|_| ()) + .unwrap_err(); + assert_eq!( + error.to_string(), + "invalid request: Bedrock doesn't support tool_choice=none. To drop it from the call, set `litellm.drop_params = True." + ); + let body = request(msgs, json!({"tool_choice": "none"}), true) + .unwrap() + .body; + assert!(body.get("toolConfig").is_none()); + } + + #[rstest] + fn unsupported_params_are_refused_or_dropped() { + let msgs = json!([{"role": "user", "content": "hi"}]); + assert!(request(msgs.clone(), json!({"seed": 3}), false).is_err()); + let body = request(msgs, json!({"seed": 3}), true).unwrap().body; + assert!(body.get("additionalModelRequestFields").is_none()); + assert_eq!(body["inferenceConfig"], json!({})); + } + + #[rstest] + fn reasoning_effort_becomes_thinking_in_additional_fields() { + let body = transform( + json!([{"role": "user", "content": "hi"}]), + json!({"reasoning_effort": "low"}), + ); + assert_eq!( + body["additionalModelRequestFields"], + json!({"thinking": {"type": "enabled", "budget_tokens": 1024}}) + ); + assert_eq!( + body["additionalModelResponseFieldPaths"], + json!(["/usage/output_tokens_details"]) + ); + assert_eq!(body["inferenceConfig"]["maxTokens"], json!(1024 + 4096)); + } + + #[rstest] + fn response_format_becomes_a_json_tool() { + let data = request( + json!([{"role": "user", "content": "hi"}]), + json!({"response_format": {"type": "json_schema", "json_schema": {"name": "x", + "schema": {"type": "object", "properties": {"a": {"type": "string"}}}}}}), + false, + ) + .unwrap(); + assert_eq!( + data.body["toolConfig"]["toolChoice"], + json!({"tool": {"name": "json_tool_call"}}) + ); + assert!(data.stream_shape.json_mode); + let response = transform_response_with( + json!({"output": {"message": {"content": [ + {"toolUse": {"toolUseId": "t", "name": "json_tool_call", "input": {"properties": {"a": "b"}}}} + ]}}, "stopReason": "tool_use", "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}}), + &data.stream_shape, + ) + .unwrap(); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("{\"a\": \"b\"}") + ); + assert_eq!(response.choices[0].finish_reason, "stop"); + } + + #[rstest] + fn orphaned_tool_calls_are_sent_as_text() { let body = transform( json!([ - {"role": "user", "content": "one"}, - {"role": "user", "content": [{"type": "text", "text": "two"}]}, - {"role": "assistant", "content": "ack"}, - {"role": "user", "content": "three"} + {"role": "user", "content": "hi"}, + {"role": "assistant", "tool_calls": [{"id": "c", "type": "function", "function": {"name": "f", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "c", "content": "ok"} ]), json!({}), ); assert_eq!( - body["messages"], - json!([ - {"role": "user", "content": [{"text": "one"}, {"text": "two"}]}, - {"role": "assistant", "content": [{"text": "ack"}]}, - {"role": "user", "content": [{"text": "three"}]} - ]) - ); - } - - #[test] - fn declines_streaming() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": true}) - ), - Some(Unsupported("streaming")) - ); - } - - #[test] - fn declines_top_k_because_python_routes_it_by_base_model() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"topK": 40}) - ), - Some(Unsupported("unrecognized request parameter")) + body["messages"][1]["content"][0]["text"], + "[tool call c: f({})]" ); } #[rstest] - #[case::tools(json!({"tools": []}))] - #[case::tool_choice(json!({"tool_choice": {"auto": {}}}))] - #[case::thinking(json!({"thinking": {"type": "enabled"}}))] - #[case::request_metadata(json!({"requestMetadata": {"k": "v"}}))] - #[case::output_config(json!({"outputConfig": {}}))] - #[case::parallel_tool_use_config(json!({"_parallel_tool_use_config": {}}))] - fn declines_tools_and_other_params_outside_the_allowlist(#[case] param: Value) { + fn tool_use_and_reasoning_come_back_as_openai_fields() { + let shape = StreamShape { + tool_name_reverse_map: StdHashMap::from([("get_weather".into(), "get.weather".into())]), + ..StreamShape::default() + }; + let response = transform_response_with( + json!({"output": {"message": {"content": [ + {"reasoningContent": {"reasoningText": {"text": "hmm", "signature": "sig"}}}, + {"text": "checking"}, + {"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}} + ]}}, "stopReason": "tool_use", "usage": {"inputTokens": 1, "outputTokens": 1}}), + &shape, + ) + .unwrap(); + let message = serde_json::to_value(&response.choices[0].message).unwrap(); assert_eq!( - reason(json!([{"role": "user", "content": "hi"}]), param.clone()), - Some(Unsupported("unrecognized request parameter")), - "expected {param} to decline" + message["tool_calls"], + json!([{"id": "t1", "type": "function", "function": {"name": "get.weather", "arguments": "{\"city\": \"Paris\"}"}, "index": 2}]) ); + assert_eq!(message["reasoning_content"], "hmm"); + assert_eq!( + message["thinking_blocks"], + json!([{"type": "thinking", "thinking": "hmm", "signature": "sig"}]) + ); + assert_eq!(response.choices[0].finish_reason, "tool_calls"); } #[rstest] - #[case::empty_string(json!(""))] - #[case::whitespace_string(json!(" "))] - #[case::whitespace_text_block(json!([{"type": "text", "text": " "}]))] - fn declines_blank_text_rather_than_substituting_the_anthropic_placeholder( - #[case] content: Value, + #[case("ab", Some("a"), "b")] + #[case("plain", None, "plain")] + fn leading_think_tags_are_reasoning( + #[case] text: &str, + #[case] reasoning: Option<&str>, + #[case] answer: &str, ) { - assert_eq!( - reason( - json!([{"role": "user", "content": content}, {"role": "user", "content": "hi"}]), - json!({}) - ), - Some(Unsupported("blank message text")), - "expected blank content {content} to decline" - ); - } - - #[test] - fn declines_a_message_whose_content_list_is_empty() { - // The blank-text check scans parts, so an empty list clears it; Converse - // rejects an empty `content` array, which is a decline the core owes the - // host before the call rather than an error after it. - assert_eq!( - reason(json!([{"role": "user", "content": []}]), json!({})), - Some(Unsupported("message without content")) - ); - assert_eq!( - reason( - json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]), - json!({}) - ), - None - ); - } - - #[test] - fn declines_a_conversation_that_opens_or_closes_on_an_assistant_turn() { - assert_eq!( - reason( - json!([ - {"role": "assistant", "content": "prefill"}, - {"role": "user", "content": "hi"} - ]), - json!({}) - ), - Some(Unsupported( - "conversation does not run user turn to user turn" - )) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "prefill"} - ]), - json!({}) - ), - Some(Unsupported( - "conversation does not run user turn to user turn" - )) - ); - } - - #[test] - fn accepts_a_user_to_user_text_conversation() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "hello"}, - {"role": "user", "content": "again"} - ]), - json!({"maxTokens": 16}) - ), - None - ); + let (found, rest) = super::parse_content_for_reasoning(text); + assert_eq!(found.as_deref(), reasoning); + assert_eq!(rest, answer); } #[test] @@ -822,24 +1297,6 @@ mod tests { assert_eq!(response.usage.total_tokens, 14); } - #[test] - fn declines_a_cache_control_message_so_widening_the_gate_is_a_red_test() { - // Converse only reports cache token counts when the request carries a - // cachePoint block, which is why the provider total and the computed one - // cannot disagree today. This is the tripwire: whoever widens the gate to - // admit prompt caching has to come back and re-check the usage mapping - // rather than discovering a silent number change in production. - assert_eq!( - reason( - json!([{"role": "user", "content": [ - {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) - ); - } - #[test] fn folds_converse_cache_tokens_into_prompt_tokens() { let response = transform_response(json!({ @@ -862,19 +1319,6 @@ mod tests { assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10); } - #[test] - fn declines_a_response_carrying_a_tool_use_block() { - let err = transform_response(json!({ - "output": {"message": {"content": [ - {"toolUse": {"toolUseId": "t1", "name": "f", "input": {}}} - ]}}, - "stopReason": "tool_use", - "usage": {"inputTokens": 1, "outputTokens": 1} - })) - .expect_err("tool use block"); - assert_eq!(err, Error::Unsupported("non-text response content block")); - } - #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( @@ -908,7 +1352,7 @@ mod tests { #[test] fn accepts_aws_call_configuration_without_serializing_it() { let call_config = json!({ - "maxTokens": 16, + "max_tokens": 16, "aws_access_key_id": "AKIA", "aws_secret_access_key": "secret", "aws_session_token": "token", @@ -921,19 +1365,12 @@ mod tests { "aws_external_id": "ext", "aws_bedrock_runtime_endpoint": "https://vpce.internal" }); - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - call_config.clone() - ), - None - ); let body = transform(json!([{"role": "user", "content": "hi"}]), call_config); assert_eq!( body, json!({ - "inferenceConfig": {"maxTokens": 16}, - "messages": [{"role": "user", "content": [{"text": "hi"}]}] + "messages": [{"role": "user", "content": [{"text": "hi"}]}], + "inferenceConfig": {"maxTokens": 16} }), "aws call configuration must not reach the Converse body" ); diff --git a/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs b/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs index d5ae8d6a323..3ec0caed6ae 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs @@ -23,9 +23,13 @@ use crate::{ anthropic::chat::handler::ModelResponseIterator, base_llm::{ base_model_iterator::StreamTransformer, - chat::streaming::{ChatStream, StreamShape}, + chat::{ + params::RESPONSE_FORMAT_TOOL_NAME, + streaming::{ChatStream, StreamShape}, + }, messages::streaming::{ByteStream, EventStream}, }, + bedrock::chat::converse_transformation::unwrap_bedrock_properties, }; #[derive(Deserialize)] @@ -200,7 +204,13 @@ pub fn converse_event_stream(bytes: ByteStream) -> BoxStream<'static, Result, + /// The block in progress is the `json_tool_call` tool a `response_format` became. + in_response_format_tool: bool, + /// Some `json_tool_call` block has been streamed back as text. + converted_response_format_tool: bool, } #[derive(Default)] @@ -233,6 +247,8 @@ impl ConverseStreamIterator { thinking_ran: false, thinking_block_index: -1, thinking_content_block_index: None, + in_response_format_tool: false, + converted_response_format_tool: false, } } @@ -295,7 +311,15 @@ impl ConverseStreamIterator { fn start(&mut self, start: &Value, index: u64) -> ConverseParsed { let mut parsed = ConverseParsed::default(); self.content_blocks.clear(); + self.in_response_format_tool = false; if let Some(tool) = start.get("toolUse").filter(|tool| !tool.is_null()) { + if self.shape.json_mode + && tool.get("name").and_then(Value::as_str) == Some(RESPONSE_FORMAT_TOOL_NAME) + { + self.in_response_format_tool = true; + self.converted_response_format_tool = true; + return parsed; + } self.tool_calls_index = Some(self.tool_calls_index.map_or(0, |index| index + 1)); let name = tool.get("name").and_then(Value::as_str).map(|name| { self.shape @@ -329,6 +353,9 @@ impl ConverseStreamIterator { .unwrap_or_default() .to_owned(); self.content_blocks.push(input.clone()); + if self.in_response_format_tool { + return parsed; + } parsed.tool_use = Some(self.tool_call(None, None, input, index)); } else if let Some(reasoning) = delta.get("reasoningContent") { parsed @@ -359,6 +386,13 @@ impl ConverseStreamIterator { self.start(start, index) } else if let Some(delta) = data.get("delta") { self.delta(delta, index) + } else if data.get("contentBlockIndex").is_some() && self.in_response_format_tool { + self.in_response_format_tool = false; + let answer = std::mem::take(&mut self.content_blocks).concat(); + ConverseParsed { + text: unwrap_bedrock_properties(if answer.is_empty() { "{}" } else { &answer }), + ..ConverseParsed::default() + } } else if data.get("contentBlockIndex").is_some() { // A block's stop: a tool that streamed only empty fragments closes with `{}`. let empty = @@ -368,10 +402,15 @@ impl ConverseStreamIterator { ..ConverseParsed::default() } } else if let Some(reason) = data.get("stopReason") { + let mut finish = finish_reason_for(reason.as_str().unwrap_or("stop")); + if finish == "tool_calls" + && self.converted_response_format_tool + && self.tool_calls_index.is_none() + { + finish = "stop"; + } ConverseParsed { - finish_reason: Some( - finish_reason_for(reason.as_str().unwrap_or("stop")).to_owned(), - ), + finish_reason: Some(finish.to_owned()), ..ConverseParsed::default() } } else if let Some(usage) = data.get("usage") { @@ -735,6 +774,39 @@ mod tests { assert_eq!(chunks[7]["choices"][0]["finish_reason"], "tool_calls"); } + #[test] + fn a_response_format_tool_streams_back_as_text() { + let mut iterator = ConverseStreamIterator::new(StreamShape { + json_mode: true, + ..StreamShape::default() + }); + let chunks: Vec = serde_json::json!([ + {"contentBlockIndex": 0, "start": {"toolUse": {"toolUseId": "t1", "name": "json_tool_call"}}}, + {"contentBlockIndex": 0, "delta": {"toolUse": {"input": "{\"properties\": {\"a\""}}}, + {"contentBlockIndex": 0, "delta": {"toolUse": {"input": ": 1}}"}}}, + {"contentBlockIndex": 0}, + {"stopReason": "tool_use"} + ]) + .as_array() + .unwrap() + .iter() + .flat_map(|event| iterator.transform(event.clone()).unwrap()) + .map(|chunk| serde_json::to_value(chunk).unwrap()) + .collect(); + for chunk in &chunks { + assert_eq!( + chunk["choices"][0]["delta"]["tool_calls"], + serde_json::Value::Null + ); + } + let content: String = chunks + .iter() + .filter_map(|chunk| chunk["choices"][0]["delta"]["content"].as_str()) + .collect(); + assert_eq!(content, "{\"a\": 1}"); + assert_eq!(chunks[4]["choices"][0]["finish_reason"], "stop"); + } + #[test] fn converse_reasoning_streams_as_thinking_blocks() { let chunks = converse_chunks(serde_json::json!([ diff --git a/litellm-rust/crates/llms/src/bedrock/chat/mod.rs b/litellm-rust/crates/llms/src/bedrock/chat/mod.rs index a46514aa697..e35bd6d88e7 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/mod.rs @@ -1,2 +1,4 @@ +pub mod converse_messages; +pub mod converse_params; pub mod converse_transformation; pub mod invoke_handler; diff --git a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs index 5ca084cd517..d8d580f3074 100644 --- a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs @@ -16,9 +16,13 @@ use crate::{ Error, base_llm::{ auth::AuthScheme, - chat::transformation::{ - BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, - ValidatedEnvironment, + chat::{ + params::ChatRequestContext, + streaming::StreamShape, + transformation::{ + BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, + ValidatedEnvironment, + }, }, }, openai_like::common_utils::{complete_openai_like_url, openai_compatible_provider_info}, @@ -90,6 +94,7 @@ impl BaseConfig for OpenAILikeChatConfig { model: &str, messages: Vec, optional_params: Map, + _context: &ChatRequestContext<'_>, ) -> Result { let mut params = Map::from_iter( optional_params @@ -113,6 +118,7 @@ impl BaseConfig for OpenAILikeChatConfig { Ok(ProviderChatRequestData { body: Value::Object(body), stream_shape: Default::default(), + headers: Vec::new(), }) } @@ -120,6 +126,7 @@ impl BaseConfig for OpenAILikeChatConfig { &self, model: &str, response: ProviderChatResponseData, + _shape: &StreamShape, ) -> Result { let mut body = response.body; sanitize_usage(&mut body); @@ -266,6 +273,7 @@ fn normalize_choice(position: usize, choice: &Value) -> Result Value { OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .transform_request(model, messages(msgs), params(opts)) + .transform_request(model, messages(msgs), params(opts), &Default::default()) .expect("request transforms") .body } fn transform_response(body: Value) -> Result { - OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .transform_response("some-model", ProviderChatResponseData { body }) + OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG.transform_response( + "some-model", + ProviderChatResponseData { body }, + &Default::default(), + ) } fn reason(msgs: Value, opts: Value) -> Option { diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs index 5ad9f8df8b8..dd2c079082c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs @@ -1,5 +1,6 @@ -use std::convert::Infallible; +use std::{convert::Infallible, sync::Arc}; +use super::model_cost::PythonModelCost; use crate::routes::inference::InferenceHost; use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned}; use litellm_inference_chat::{Error, route::ChatCompletions, types::ChatCompletionsCall}; @@ -26,6 +27,10 @@ pub(super) fn project( custom_llm_provider: call.options.custom_llm_provider, extra_headers: call.options.extra_headers, timeout: call.options.timeout, + // Python's own settings decide what a drop would do, so a param that needs one goes + // back to Python. Capabilities come from the cost map Python itself reads. + drop_params: None, + model_info: Some(Arc::new(PythonModelCost::current(py)?)), }) } diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs index d78d141f0f2..ce319189a60 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs @@ -1,4 +1,5 @@ mod host; +mod model_cost; use std::sync::Arc; diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/model_cost.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/model_cost.rs new file mode 100644 index 00000000000..09a7e64f101 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/model_cost.rs @@ -0,0 +1,88 @@ +//! Python's `litellm.model_cost` as the cost map chat translation reads, so a capability +//! lookup sees the map Python itself would, models registered at runtime included. + +use std::{borrow::Cow, collections::HashSet, sync::OnceLock}; + +use litellm_host_python::from_py; +use litellm_llms::base_llm::model_info::ModelInfo; +use pyo3::{prelude::*, types::PyDict}; +use serde_json::{Map, Value}; + +pub(super) struct PythonModelCost { + map: Py, + declared: OnceLock>, +} + +impl PythonModelCost { + /// The dict `litellm.model_cost` holds now. A map Python replaces later is not followed, + /// as a call already under way would not follow it either. + pub(super) fn current(py: Python<'_>) -> PyResult { + let map = py + .import("litellm")? + .getattr("model_cost")? + .cast_into::()? + .unbind(); + Ok(Self { + map, + declared: OnceLock::new(), + }) + } +} + +impl ModelInfo for PythonModelCost { + fn entry(&self, model: &str) -> Option>> { + Python::attach(|py| { + let entry = self.map.bind(py).get_item(model).ok()??; + from_py::>(&entry).ok().map(Cow::Owned) + }) + } + + fn declares(&self, key: &str) -> bool { + self.declared + .get_or_init(|| { + Python::attach(|py| { + self.map + .bind(py) + .values() + .iter() + .filter_map(|entry| entry.cast_into::().ok()) + .flat_map(|entry| { + entry + .keys() + .iter() + .filter_map(|key| key.extract::().ok()) + .collect::>() + }) + .collect() + }) + }) + .contains(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_entry_python_holds_and_the_keys_any_entry_sets() { + Python::initialize(); + Python::attach(|py| { + let map = PyDict::new(py); + let entry = PyDict::new(py); + entry.set_item("supports_reasoning", true).unwrap(); + entry.set_item("max_output_tokens", 64000).unwrap(); + map.set_item("claude-x", entry).unwrap(); + let cost = PythonModelCost { + map: map.unbind(), + declared: OnceLock::new(), + }; + let found = cost.entry("claude-x").unwrap(); + assert_eq!(found.get("supports_reasoning"), Some(&Value::Bool(true))); + assert_eq!(found.get("max_output_tokens"), Some(&Value::from(64000))); + assert!(cost.entry("unknown").is_none()); + assert!(cost.declares("max_output_tokens")); + assert!(!cost.declares("default_reasoning_effort")); + }); + } +} diff --git a/tests/test_litellm_rust/chat_completions/test_callbacks.py b/tests/test_litellm_rust/chat_completions/test_callbacks.py index 6743f9adbf0..ba5461bd24f 100644 --- a/tests/test_litellm_rust/chat_completions/test_callbacks.py +++ b/tests/test_litellm_rust/chat_completions/test_callbacks.py @@ -22,13 +22,13 @@ _OBJECT: Final = TypeAdapter(dict[str, object]) async def test_returns_a_model_response_and_logs_success_once(server: RecordingServer, asynchronous: bool) -> None: recorder: Final = RecordingLogger() - result: Final = await call(server, asynchronous=asynchronous, callbacks=[recorder], temperature=0.25) + result: Final = await call(server, asynchronous=asynchronous, callbacks=[recorder], temperature=1) assert_served_natively(server) sent: Final = server.requests[0] body: Final = _OBJECT.validate_python(sent.body) assert sent.path == "/v1/messages" - assert body["temperature"] == 0.25 + assert body["temperature"] == 1 assert request_body(_OBJECT.validate_python(recorder.wait_for("log_pre_api_call")[0].kwargs)) == body assert isinstance(result, ModelResponse) assert result.choices[0].message.content == MESSAGES_TEXT diff --git a/tests/test_litellm_rust/chat_completions/test_requests.py b/tests/test_litellm_rust/chat_completions/test_requests.py index c8800501cd4..41aa9b33932 100644 --- a/tests/test_litellm_rust/chat_completions/test_requests.py +++ b/tests/test_litellm_rust/chat_completions/test_requests.py @@ -18,7 +18,8 @@ _OBJECT: Final = TypeAdapter(dict[str, object]) @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", (False, True)) async def test_positional_parameters_reach_the_wire(server: RecordingServer, asynchronous: bool) -> None: - temperature: Final = 0.35 + # The model takes no sampling params but temperature=1, as Python itself rules. + temperature: Final = 1.0 arguments: Final = ( (MESSAGES_MODEL, list(MESSAGES), None, None, 12.0, temperature) if asynchronous