From 3d976a5791c816fac60868a7eedaa88c53926418 Mon Sep 17 00:00:00 2001 From: Nate Armstrong Date: Mon, 19 Oct 2026 10:44:12 -0700 Subject: [PATCH] feat(gateway): full openai chat request translation for anthropic and bedrock (#47088) * feat(gateway): share router cooldowns and follow proxy key invalidations through Redis Gateway replicas now share state through the coordination Redis the Python proxy uses (general_settings.coordination_redis, a redis response cache, or REDIS_*): - A deployment one replica cools down is written under Python's deployment::cooldown key and applied by every other replica's background poll (one MGET a second), so the fleet stops sending to a failing deployment together. allowed_fails counts failures across replicas, as Python's v1 rule does; the v2 failure-rate rule stays per replica, as in Python. Deployments are named by model_info.id, or an id derived from the config that is the same on every replica. - The virtual key cache follows the proxy's litellm_proxy.auth_cache_invalidation channel, so a key blocked, deleted or edited through the admin API (or its team or user) is re-read on every replica at once instead of after up to 60 s. general_settings.user_api_key_cache_ttl sets the key TTL. Nothing is added to the path of a successful request. Without Redis, or while it is down, each replica keeps its own state as before. Redis Cluster and Sentinel are refused at startup for now. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): hooks and observers around inference requests, with usage and cost Adds the litellm-gateway-hooks crate: request facts built only when something reads them, on-path hooks that declare what they read and what they may change and run in config order, off-path observers on bounded queues, a usage tap for SSE and JSON responses, and per-deployment prices from the model cost map overlaid by config prices. The messages and chat handlers run through it when anything is installed; with nothing installed requests take the old path. The key store now also returns the key alias, user, team alias and key and team metadata, and router attempts carry the deployment id. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): write spend to the proxy's database from a native observer Adds a spend writer observer that records every gateway request in the proxy's own Postgres tables with the same statements the Python proxy uses: spend logs, key/user/team/team member/end user spend, and the daily user, team and end-user activity tables. Writes are batched on the proxy's timers and are additive, so replicas and Python proxies can write side by side. The database pool is now shared between virtual keys and the writer. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): enforce key, team, user, member and end-user budgets on shared spend counters Adds a built-in budgets hook that runs the proxy's budget checks, in its order and with its messages, on the proxy's Redis spend:* counters. Each request reads and seeds the counters, checks them and reserves its maximum cost in one Redis script, then settles the reservation to its real cost off the request path, so replicas and Python proxies sharing the Redis enforce one budget. Without Redis the counters are kept per replica. The key store now reads user, team member and end-user budgets, and auth no longer checks budgets against stored spend. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): enforce key, user, team, member and end-user rate limits on shared counters Adds a built-in rate limits hook with the proxy's v3 limiter semantics: requests per window, tokens per window and in-flight requests per key, on the proxy's own {scope:id}:window|requests|tokens|max_parallel_requests Redis keys and with its scripts, so replicas and Python proxies sharing the Redis enforce one limit. Tokens are reserved from the proxy's estimate and settled to usage when the response ends. Refusals carry the proxy's message, body and headers on each endpoint, and complete responses carry its x-ratelimit-* headers. Without Redis the counters are kept per replica. The key store now reads key, user, team and member rpm/tpm limits and the key's max_parallel_requests, and end users' budget limits are read with their budgets; keys with plain rate limits are no longer refused. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): configurable header, spend metadata and provider metadata hooks Adds four built-in hooks a config lists under litellm_settings.hooks and runs in that order: forward_headers copies named client headers to the provider, strip_headers removes headers from the provider request (including the gateway's default pass-through), spend_metadata adds header and key-metadata values to the spend log's spend_logs_metadata, and provider_metadata sets Messages body fields and headers from templates over request facts. Hooks' upstream header edits now reach the provider on both endpoints, and body edits can set nested fields. Credentials and signer headers are never forwarded or set, and a misconfigured hook stops the gateway at startup. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): model capacity hook, the proxy's dynamic rate limiter Adds the model_capacity hook, turned on by listing dynamic_rate_limiter_v3 (or model_capacity) in litellm_settings.callbacks. It caps each model group at the sum of its deployments' rpm and tpm across every replica, and with priority_reservation holds each priority's pool to its share once the group's saturation reaches the threshold, counting it below. It uses the proxy's model_saturation_check and priority_model keys and scripts, its refusal messages and headers, and adds a successful response's tokens to the group and the pool admission chose. The rate limit counters gain a general check-and-add over request and token pairs, and the throttling refusal shape is shared. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * test(gateway-auth): expect the narrowed unsupported rate limit message Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): prometheus observer with the proxy's metric and label names Adds the prometheus observer, turned on by listing prometheus in litellm_settings.callbacks, success_callback or failure_callback. Each finished request updates the proxy's request, spend, token, latency and deployment metrics under its names and labels, and /metrics serves them after the gateway's own. prometheus_metrics_config, the exclude lists and prometheus_latency_buckets are read as the proxy reads them. The metrics the gateway already served under the same names come from the observer alone when it is on. Outcomes now carry when the attempt was sent upstream, for the LLM API latency and time to first token. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): s3_v2 observer writing the proxy's logging payload to S3 Adds the s3_v2 observer, turned on by listing s3_v2 (or s3_log) in litellm_settings.callbacks, success_callback or failure_callback. Each finished request becomes the proxy's StandardLoggingPayload, with the messages from the request body and the response as the client got it, streams put back together. It is queued and written every flush interval under the proxy's key layout, with its Content-MD5, Content-Disposition and encryption headers, one PUT per request or one NDJSON object per folder with s3_batch_file_upload. s3_callback_params is read as the proxy reads it, os.environ references included. Uploads are retried, dropped and kept for the next flush by the proxy's rules, and what is queued is written at shutdown. auth-aws gains an S3 signer that signs the path as sent and every header, as botocore does. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_0138knkqKpkoSr6PApenZan5 * feat(gateway): Azure OpenAI chat completions through the OpenAI passthrough Adds azure/ chat completions, streaming and not, on the same passthrough the openai/ route uses. The request goes to the deployment path with the api-version query, or to /openai/v1/ for the v1, latest and preview versions, with api_base, api_key and api_version falling back to the AZURE_* variables and os.environ references resolved as the proxy does. A key is sent as api-key, and an azure_ad_token or an Entra ID token from auth-azure goes as a Bearer token. The body follows the proxy's gpt, o-series and gpt-5 param rules, picked by base_model or the deployment name and reading model capabilities from the cost map, so an unsupported param is refused unless drop_params allows it to be dropped. Spend is priced as the proxy prices Azure, by azure/ when one is set and otherwise by the model the response names, and a content filter error keeps Azure's inner error for the client * feat(gateway): Anthropic and Bedrock chat completions streaming Adds "stream": true to /v1/chat/completions for anthropic/ and bedrock/ deployments, which were refused with 501 before. The request is the non-streaming one with "stream": true added for Anthropic and sent to /converse-stream for Bedrock, and the provider's events are decoded the way the proxy's chunk parsers decode them: text, tool call deltas with empty arguments sent as "{}", reasoning_content and thinking_blocks with their index and signature, citations and the mapped finish_reason. A wrapper in the shape of the proxy's stream wrapper gives every chunk one id and the client's model name, sends role on the first chunk, adds the empty finish chunk, strips usage from the chunks and, when stream_options.include_usage is set, ends with the usage chunk before [DONE]. The gateway waits for the first chunk before answering, so an upstream error up to there is retried or falls back to another deployment, and an error after it ends the stream with the OpenAI error frame. Usage, cache reads and writes included, is handed to the meter through the response whether or not the client asked for it, and dropping the client stream drops the upstream request * feat(gateway): full OpenAI chat request translation for Anthropic and Bedrock Anthropic and Bedrock Converse deployments now take the whole OpenAI chat request the Python proxy accepts instead of a narrow allowlist. Messages translate as Python builds them: system as a string, parts or several messages, text, image_url (URL or data URL), file and document parts, assistant tool_calls, the tool and function roles, consecutive role merging, empty content, cache_control on Anthropic and cachePoint on Bedrock. Params map per provider: max_tokens and max_completion_tokens with Anthropic's model default, temperature, top_p, stop, tools and tool_choice, parallel_tool_calls, response_format as a json tool or native output format, reasoning_effort and thinking, user as metadata, with n and any other unsupported param refused with a 400 unless drop_params is set globally, on the deployment or on the request, and allowed_openai_params and additional_drop_params honoured. Responses carry tool calls, thinking blocks and response_format answers back in OpenAI shape, streamed and not. The route downloads media URLs a provider cannot take by reference, merges the beta headers a request needs with forwarded ones, and the gateway hands the cost map to both providers for the capability rules that read it. * fix(rust-bridge): read Python's model_cost for chat translation capabilities Chat translation for Anthropic and Bedrock reads capability flags from the cost map. The Python bridge passed none, so every SDK chat call through the Rust path failed. It now reads litellm.model_cost live, so lookups match Python's, runtime-registered models included. ModelInfo::entry returns a Cow so a map read from Python can hand over a copy. Two tests sent temperature != 1 to a model that removed sampling params, which Python itself refuses; they now send temperature=1. * fix(model-catalog): read token counts the cost map writes as whole floats The published cost map now has prompt_cache_min_tokens: 4096.0, which the typed u64 fields rejected, so the gateway dropped the whole map at startup and priced every request at zero. Counts accept any whole non-negative number, as Python reads any JSON number, and a test keeps the checked-in map parsing. * feat(gateway): Azure OpenAI chat completions through the OpenAI passthrough (#47074) Adds azure/ chat completions, streaming and not, on the same passthrough the openai/ route uses. The request goes to the deployment path with the api-version query, or to /openai/v1/ for the v1, latest and preview versions, with api_base, api_key and api_version falling back to the AZURE_* variables and os.environ references resolved as the proxy does. A key is sent as api-key, and an azure_ad_token or an Entra ID token from auth-azure goes as a Bearer token. The body follows the proxy's gpt, o-series and gpt-5 param rules, picked by base_model or the deployment name and reading model capabilities from the cost map, so an unsupported param is refused unless drop_params allows it to be dropped. Spend is priced as the proxy prices Azure, by azure/ when one is set and otherwise by the model the response names, and a content filter error keeps Azure's inner error for the client * feat(gateway): Anthropic and Bedrock chat completions streaming (#47077) Adds "stream": true to /v1/chat/completions for anthropic/ and bedrock/ deployments, which were refused with 501 before. The request is the non-streaming one with "stream": true added for Anthropic and sent to /converse-stream for Bedrock, and the provider's events are decoded the way the proxy's chunk parsers decode them: text, tool call deltas with empty arguments sent as "{}", reasoning_content and thinking_blocks with their index and signature, citations and the mapped finish_reason. A wrapper in the shape of the proxy's stream wrapper gives every chunk one id and the client's model name, sends role on the first chunk, adds the empty finish chunk, strips usage from the chunks and, when stream_options.include_usage is set, ends with the usage chunk before [DONE]. The gateway waits for the first chunk before answering, so an upstream error up to there is retried or falls back to another deployment, and an error after it ends the stream with the OpenAI error frame. Usage, cache reads and writes included, is handed to the meter through the response whether or not the client asked for it, and dropping the client stream drops the upstream request --------- Co-authored-by: Claude Opus 5.5 (1M context) --- litellm-rust/Cargo.lock | 3 + .../gateway-inference/src/chat_completions.rs | 20 +- .../crates/gateway-inference/src/lib.rs | 12 +- .../crates/gateway-inference/tests/routes.rs | 2 + litellm-rust/crates/gateway/src/hooks.rs | 30 +- .../crates/gateway/tests/chat_translation.rs | 722 ++++++++ litellm-rust/crates/inference-chat/Cargo.toml | 1 + .../crates/inference-chat/src/handler.rs | 6 +- litellm-rust/crates/inference-chat/src/lib.rs | 15 +- .../crates/inference-chat/src/media.rs | 158 ++ .../crates/inference-chat/src/prepare.rs | 76 +- .../crates/inference-chat/src/route.rs | 2 + .../crates/inference-chat/src/stream.rs | 7 +- .../crates/inference-chat/src/types.rs | 27 +- .../crates/inference-chat/tests/caching.rs | 6 + .../inference-chat/tests/chat_completions.rs | 6 +- .../src/formats/chat_completions/response.rs | 9 + litellm-rust/crates/llms/Cargo.toml | 2 + .../crates/llms/src/anthropic/chat/handler.rs | 86 +- .../llms/src/anthropic/chat/messages.rs | 777 ++++++++ .../crates/llms/src/anthropic/chat/mod.rs | 2 + .../crates/llms/src/anthropic/chat/params.rs | 1293 +++++++++++++ .../llms/src/anthropic/chat/transformation.rs | 1632 ++++++++++++----- .../llms/src/azure/chat/transformation.rs | 104 +- .../crates/llms/src/base_llm/chat/media.rs | 209 +++ .../crates/llms/src/base_llm/chat/mod.rs | 3 + .../crates/llms/src/base_llm/chat/params.rs | 434 +++++ .../llms/src/base_llm/chat/response_format.rs | 445 +++++ .../llms/src/base_llm/chat/streaming.rs | 8 + .../llms/src/base_llm/chat/transformation.rs | 23 +- .../crates/llms/src/base_llm/model_info.rs | 14 +- .../src/bedrock/chat/converse_messages.rs | 1343 ++++++++++++++ .../llms/src/bedrock/chat/converse_params.rs | 1039 +++++++++++ .../bedrock/chat/converse_transformation.rs | 1151 ++++++++---- .../llms/src/bedrock/chat/invoke_handler.rs | 82 +- .../crates/llms/src/bedrock/chat/mod.rs | 2 + .../src/openai_like/chat/transformation.rs | 23 +- .../src/routes/chat_completions/host.rs | 7 +- .../src/routes/chat_completions/mod.rs | 1 + .../src/routes/chat_completions/model_cost.rs | 88 + .../chat_completions/test_callbacks.py | 4 +- .../chat_completions/test_requests.py | 3 +- 42 files changed, 8944 insertions(+), 933 deletions(-) create mode 100644 litellm-rust/crates/gateway/tests/chat_translation.rs create mode 100644 litellm-rust/crates/inference-chat/src/media.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/chat/messages.rs create mode 100644 litellm-rust/crates/llms/src/anthropic/chat/params.rs create mode 100644 litellm-rust/crates/llms/src/base_llm/chat/media.rs create mode 100644 litellm-rust/crates/llms/src/base_llm/chat/params.rs create mode 100644 litellm-rust/crates/llms/src/base_llm/chat/response_format.rs create mode 100644 litellm-rust/crates/llms/src/bedrock/chat/converse_messages.rs create mode 100644 litellm-rust/crates/llms/src/bedrock/chat/converse_params.rs create mode 100644 litellm-rust/crates/python-bridge/src/routes/chat_completions/model_cost.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 4b741c993e4..1c4e6553b66 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4055,6 +4055,7 @@ dependencies = [ name = "litellm-inference-chat" version = "0.1.0" dependencies = [ + "base64 0.22.1", "bytes", "futures-util", "litellm-auth", @@ -4235,11 +4236,13 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_with", + "sha2 0.10.9", "strum", "thiserror 2.0.19", "time", "tokio", "url", + "uuid", ] [[package]] diff --git a/litellm-rust/crates/gateway-inference/src/chat_completions.rs b/litellm-rust/crates/gateway-inference/src/chat_completions.rs index 6f4442dd053..84abff5fe70 100644 --- a/litellm-rust/crates/gateway-inference/src/chat_completions.rs +++ b/litellm-rust/crates/gateway-inference/src/chat_completions.rs @@ -146,6 +146,8 @@ async fn handle( custom_llm_provider: deployment.custom_llm_provider.clone(), extra_headers: extra_headers.clone(), timeout: deployment.timeout, + drop_params: Some(drop_params(gateway, deployment)), + model_info: gateway.model_info(), }; if stream { return translated_stream(route, call, name).await; @@ -197,6 +199,17 @@ fn provider_params(deployment: &Deployment) -> Map { provider_params } +/// `litellm.drop_params` or the deployment's own: either one set drops what the provider +/// cannot take, as Python's `get_optional_params` reads them. +fn drop_params(gateway: &Gateway, deployment: &Deployment) -> bool { + gateway.drop_params() + || deployment + .litellm_params + .drop_params + .as_ref() + .is_some_and(|flag| flag.is_set()) +} + async fn openai( gateway: &Gateway, deployment: &Deployment, @@ -213,12 +226,7 @@ async fn openai( extra_headers, timeout: deployment.timeout, litellm_params: provider_params(deployment), - drop_params: gateway.drop_params() - || deployment - .litellm_params - .drop_params - .as_ref() - .is_some_and(|flag| flag.is_set()), + drop_params: drop_params(gateway, deployment), model_info: gateway.model_info(), }, None, diff --git a/litellm-rust/crates/gateway-inference/src/lib.rs b/litellm-rust/crates/gateway-inference/src/lib.rs index 65aa2d72779..010e92cfbe1 100644 --- a/litellm-rust/crates/gateway-inference/src/lib.rs +++ b/litellm-rust/crates/gateway-inference/src/lib.rs @@ -18,7 +18,10 @@ mod responses; use std::sync::{Arc, OnceLock}; use axum::{Router, routing::post}; -use litellm_http::{ClientVariant, HttpClientConfig, media::UrlPolicy}; +use litellm_http::{ + ClientVariant, HttpClientConfig, + media::{MediaFetcher, UrlPolicy}, +}; use litellm_inference::resources::CoreResources; use litellm_inference_chat::ChatCompletionsRoute; use litellm_inference_messages::MessagesRoute; @@ -138,7 +141,12 @@ impl Gateway { provider.clone(), auth.clone(), secrets.clone(), - ), + ) + .with_media(MediaFetcher::new( + &resources.pool, + &http, + UrlPolicy::default(), + )?), messages: MessagesRoute::new(provider.clone(), auth.clone(), secrets.clone()), responses: ResponsesRoute::new(provider, auth.clone(), secrets.clone()), ocr: OcrRoute::new(OcrClient::new( diff --git a/litellm-rust/crates/gateway-inference/tests/routes.rs b/litellm-rust/crates/gateway-inference/tests/routes.rs index 73925399e38..7452306338e 100644 --- a/litellm-rust/crates/gateway-inference/tests/routes.rs +++ b/litellm-rust/crates/gateway-inference/tests/routes.rs @@ -105,6 +105,8 @@ async fn chat_errors_come_from_core( custom_llm_provider: None, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, }, &(), None, diff --git a/litellm-rust/crates/gateway/src/hooks.rs b/litellm-rust/crates/gateway/src/hooks.rs index 6c393816cbf..9f315cd3f78 100644 --- a/litellm-rust/crates/gateway/src/hooks.rs +++ b/litellm-rust/crates/gateway/src/hooks.rs @@ -1,7 +1,7 @@ //! Builds the hooks and observers inference requests run through, and the per-deployment prices //! observers that read cost need. -use std::{collections::HashSet, sync::Arc}; +use std::{borrow::Cow, collections::HashSet, sync::Arc}; use litellm_config::{Config, Model}; use litellm_gateway_hooks::{ @@ -30,19 +30,19 @@ pub async fn install( hooks: Vec>, observers: Vec>, ) -> Result<(), Error> { - let serves_azure = config.model_list.iter().any(is_azure); - if hooks.is_empty() && observers.is_empty() && !serves_azure { + let reads_model_info = config.model_list.iter().any(reads_model_info); + if hooks.is_empty() && observers.is_empty() && !reads_model_info { return Ok(()); } let needs_cost = observers .iter() .any(|observer| observer.reads().contains(Facts::COST)) || hooks.iter().any(|hook| hook.reads().contains(Facts::COST)); - let catalog = match needs_cost || serves_azure { + let catalog = match needs_cost || reads_model_info { true => load_catalog(inference).await.map(Arc::new), false => None, }; - if let Some(catalog) = catalog.as_ref().filter(|_| serves_azure) { + if let Some(catalog) = catalog.as_ref().filter(|_| reads_model_info) { inference.install_model_info(Arc::new(CostMap::new(catalog.clone()))); } if hooks.is_empty() && observers.is_empty() { @@ -204,7 +204,19 @@ fn is_azure(model: &Model) -> bool { } } -/// The model cost map as the capabilities Azure's param rules read. +/// Whether the deployment's chat param rules read model capabilities from the cost map: +/// Azure, and the Anthropic and Bedrock chat translations. +fn reads_model_info(model: &Model) -> bool { + is_azure(model) + || litellm_inference::provider::resolve_llm_provider( + &model.litellm_params.model, + model.litellm_params.custom_llm_provider.as_deref(), + "chat completions", + ) + .is_ok_and(|resolved| matches!(<&str>::from(resolved.provider), "anthropic" | "bedrock")) +} + +/// The model cost map as the capabilities the chat param rules read. struct CostMap { catalog: Arc, keys: HashSet, @@ -222,8 +234,10 @@ impl CostMap { } impl ModelInfo for CostMap { - fn entry(&self, model: &str) -> Option<&Map> { - self.catalog.lookup(model).map(|found| found.entry.fields()) + fn entry(&self, model: &str) -> Option>> { + self.catalog + .lookup(model) + .map(|found| Cow::Borrowed(found.entry.fields())) } fn declares(&self, key: &str) -> bool { diff --git a/litellm-rust/crates/gateway/tests/chat_translation.rs b/litellm-rust/crates/gateway/tests/chat_translation.rs new file mode 100644 index 00000000000..aa0c1da8f6f --- /dev/null +++ b/litellm-rust/crates/gateway/tests/chat_translation.rs @@ -0,0 +1,722 @@ +//! Chat completions for deployments the gateway translates (Anthropic, Bedrock Converse): the +//! OpenAI request widened to tools, images and the provider's own limits, the provider body +//! the upstream sees, and the OpenAI answer the client gets back, streamed and not. + +use std::{ + io::Write, + sync::{Arc, Mutex, Once}, +}; + +use axum::{ + Router, + body::{Body, Bytes, to_bytes}, + http::{Request, header::CONTENT_TYPE}, + response::IntoResponse, +}; +use litellm_config::Config; +use litellm_framer::aws_event_stream::{AwsEventStreamCodec, Header, HeaderValue, Message}; +use rstest::rstest; +use serde_json::{Value, json}; +use tokio::net::TcpListener; +use tokio_util::codec::Encoder; +use tower::ServiceExt; + +const ANTHROPIC_MODEL: &str = "claude-x"; +const BEDROCK_MODEL: &str = "anthropic.claude-x-v1:0"; +/// A one-pixel PNG. +const PNG: &str = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg=="; + +/// The gateway reads its cost map from this variable; every test sets it before building a +/// gateway, so no test reads the environment while it changes. +fn use_local_cost_map() { + static ONCE: Once = Once::new(); + ONCE.call_once(|| { + let capabilities = json!({ + "mode": "chat", + "max_tokens": 8192, + "max_output_tokens": 8192, + "supports_tool_choice": true, + "supports_function_calling": true, + "supports_vision": true, + "supports_pdf_input": true, + "supports_prompt_caching": true, + "supports_response_schema": true, + }); + let entry = |provider: &str| { + let mut entry = capabilities.clone(); + entry["litellm_provider"] = json!(provider); + entry + }; + let map = json!({ + ANTHROPIC_MODEL: entry("anthropic"), + BEDROCK_MODEL: entry("bedrock_converse"), + }); + let mut file = tempfile::NamedTempFile::new().unwrap(); + file.write_all(map.to_string().as_bytes()).unwrap(); + let (_, path) = file.keep().unwrap(); + // SAFETY: set once, before any test builds a gateway or spawns a thread that reads the + // environment. + unsafe { std::env::set_var("LITELLM_MODEL_COST_MAP_PATH", path) }; + }); +} + +fn anthropic_events(events: &[Value]) -> String { + events + .iter() + .map(|event| { + format!( + "event: {}\ndata: {event}\n\n", + event["type"].as_str().unwrap() + ) + }) + .collect() +} + +fn anthropic_tool_message() -> Value { + json!({ + "id": "msg_1", "type": "message", "role": "assistant", "model": ANTHROPIC_MODEL, + "content": [ + {"type": "text", "text": "Checking."}, + {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}} + ], + "stop_reason": "tool_use", "stop_sequence": null, + "usage": {"input_tokens": 10, "output_tokens": 5} + }) +} + +/// The answer to a `response_format` request, given as the tool it became. +fn anthropic_json_message() -> Value { + json!({ + "id": "msg_3", "type": "message", "role": "assistant", "model": ANTHROPIC_MODEL, + "content": [{"type": "tool_use", "id": "toolu_2", "name": "json_tool_call", "input": {"city": "Paris"}}], + "stop_reason": "tool_use", "stop_sequence": null, + "usage": {"input_tokens": 10, "output_tokens": 5} + }) +} + +fn anthropic_text_message() -> Value { + json!({ + "id": "msg_2", "type": "message", "role": "assistant", "model": ANTHROPIC_MODEL, + "content": [{"type": "text", "text": "Sunny."}], + "stop_reason": "end_turn", "stop_sequence": null, + "usage": {"input_tokens": 10, "output_tokens": 2} + }) +} + +fn anthropic_tool_stream() -> String { + anthropic_events(&[ + json!({"type": "message_start", "message": {"id": "msg_1", "type": "message", "role": "assistant", "model": ANTHROPIC_MODEL, "content": [], "stop_reason": null, "stop_sequence": null, "usage": {"input_tokens": 10, "output_tokens": 1}}}), + json!({"type": "content_block_start", "index": 0, "content_block": {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {}}}), + json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "{\"city\": "}}), + json!({"type": "content_block_delta", "index": 0, "delta": {"type": "input_json_delta", "partial_json": "\"Paris\"}"}}), + json!({"type": "content_block_stop", "index": 0}), + json!({"type": "message_delta", "delta": {"stop_reason": "tool_use", "stop_sequence": null}, "usage": {"output_tokens": 5}}), + json!({"type": "message_stop"}), + ]) +} + +fn converse_tool_response() -> Value { + json!({ + "output": {"message": {"role": "assistant", "content": [ + {"text": "Checking."}, + {"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather", "input": {"city": "Paris"}}} + ]}}, + "stopReason": "tool_use", + "usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15} + }) +} + +fn converse_text_response() -> Value { + json!({ + "output": {"message": {"role": "assistant", "content": [{"text": "Sunny."}]}}, + "stopReason": "end_turn", + "usage": {"inputTokens": 10, "outputTokens": 2, "totalTokens": 12} + }) +} + +fn bedrock_frame(name: &str, payload: Value, wire: &mut bytes::BytesMut) { + let message = Message::new(Bytes::from(payload.to_string())) + .add_header(Header::new( + ":message-type", + HeaderValue::String("event".to_owned().into()), + )) + .add_header(Header::new( + ":event-type", + HeaderValue::String(name.to_owned().into()), + )); + AwsEventStreamCodec.encode(message, wire).unwrap(); +} + +fn converse_tool_stream() -> Bytes { + let mut wire = bytes::BytesMut::new(); + for (name, payload) in [ + ("messageStart", json!({"role": "assistant"})), + ( + "contentBlockStart", + json!({"contentBlockIndex": 0, "start": {"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather"}}}), + ), + ( + "contentBlockDelta", + json!({"contentBlockIndex": 0, "delta": {"toolUse": {"input": "{\"city\": "}}}), + ), + ( + "contentBlockDelta", + json!({"contentBlockIndex": 0, "delta": {"toolUse": {"input": "\"Paris\"}"}}}), + ), + ("contentBlockStop", json!({"contentBlockIndex": 0})), + ("messageStop", json!({"stopReason": "tool_use"})), + ( + "metadata", + json!({"usage": {"inputTokens": 10, "outputTokens": 5, "totalTokens": 15}, "metrics": {"latencyMs": 9}}), + ), + ] { + bedrock_frame(name, payload, &mut wire); + } + wire.freeze() +} + +#[derive(Clone, Debug)] +struct Sent { + path: String, + headers: Vec<(String, String)>, + body: Value, +} + +impl Sent { + fn header(&self, name: &str) -> Option<&str> { + self.headers + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.as_str()) + } +} + +type Calls = Arc>>; + +/// Answers each provider by path: a tool call while the conversation has no tool result yet, +/// the final answer once it does. +async fn upstream(calls: Calls) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let app = Router::new().fallback(move |request: Request| { + let calls = calls.clone(); + async move { + let (parts, body) = request.into_parts(); + let body: Value = + serde_json::from_slice(&to_bytes(body, 1 << 22).await.unwrap()).unwrap(); + let path = parts.uri.path().to_owned(); + calls.lock().unwrap().push(Sent { + path: path.clone(), + headers: parts + .headers + .iter() + .map(|(name, value)| { + ( + name.to_string(), + value.to_str().unwrap_or_default().to_owned(), + ) + }) + .collect(), + body: body.clone(), + }); + let answered = + body.to_string().contains("tool_result") || body.to_string().contains("toolResult"); + let json = [(CONTENT_TYPE, "application/json")]; + if path.ends_with("/converse-stream") { + return ( + [(CONTENT_TYPE, "application/vnd.amazon.eventstream")], + converse_tool_stream(), + ) + .into_response(); + } + if path.ends_with("/converse") { + let answer = match answered { + true => converse_text_response(), + false => converse_tool_response(), + }; + return (json, answer.to_string()).into_response(); + } + if body["stream"] == json!(true) { + return ( + [(CONTENT_TYPE, "text/event-stream")], + anthropic_tool_stream(), + ) + .into_response(); + } + let answer = match answered { + _ if body["tools"][0]["name"] == "json_tool_call" => anthropic_json_message(), + true => anthropic_text_message(), + false => anthropic_tool_message(), + }; + (json, answer.to_string()).into_response() + } + }); + tokio::spawn(async move { axum::serve(listener, app).await }); + format!("http://{address}") +} + +struct Harness { + app: Router, + calls: Calls, +} + +async fn harness() -> Harness { + use_local_cost_map(); + let calls = Calls::default(); + let upstream = upstream(calls.clone()).await; + let config = Config::from_yaml(&format!( + "model_list: + - model_name: claude + litellm_params: + model: anthropic/{ANTHROPIC_MODEL} + api_base: {upstream} + api_key: sk-anthropic + - model_name: bedrock + litellm_params: + model: bedrock/{BEDROCK_MODEL} + api_base: {upstream} + api_key: bedrock-token + - model_name: claude-dropping + litellm_params: + model: anthropic/{ANTHROPIC_MODEL} + api_base: {upstream} + api_key: sk-anthropic + drop_params: true +router_settings: + num_retries: 0 +general_settings: + master_key: sk-master +" + )) + .unwrap(); + let inference = litellm_gateway::build_inference(&config).unwrap(); + litellm_gateway::hooks::install(&inference, &config, Vec::new(), Vec::new()) + .await + .unwrap(); + let app = litellm_gateway::router(inference, &config, None, None); + Harness { app, calls } +} + +impl Harness { + async fn send(&self, body: Value) -> (u16, String) { + let request = Request::post("/v1/chat/completions") + .header("content-type", "application/json") + .header("authorization", "Bearer sk-master") + .body(Body::from(body.to_string())) + .unwrap(); + let response = self.app.clone().oneshot(request).await.unwrap(); + let status = response.status().as_u16(); + let body = to_bytes(response.into_body(), 1 << 22).await.unwrap(); + (status, String::from_utf8(body.to_vec()).unwrap()) + } + + async fn json(&self, body: Value) -> Value { + let (status, text) = self.send(body).await; + assert_eq!(status, 200, "{text}"); + serde_json::from_str(&text).unwrap() + } + + fn sent(&self) -> Vec { + self.calls.lock().unwrap().clone() + } +} + +fn weather_tool() -> Value { + json!({"type": "function", "function": { + "name": "get_weather", + "description": "The weather in a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]} + }}) +} + +fn question() -> Value { + json!({"role": "user", "content": "Weather in Paris?"}) +} + +/// The conversation after the model asked for the tool and the tool answered. +fn answered(call_id: &str) -> Value { + json!([ + question(), + {"role": "assistant", "content": null, "tool_calls": [ + {"id": call_id, "type": "function", "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}} + ]}, + {"role": "tool", "tool_call_id": call_id, "content": "sunny, 21C"} + ]) +} + +/// The streamed chunks' JSON and whether the stream ended with `[DONE]`. +fn frames(body: &str) -> (Vec, bool) { + let mut done = false; + let frames = body + .split("\n\n") + .filter_map(|frame| frame.strip_prefix("data: ")) + .filter(|data| { + let last = *data == "[DONE]"; + done |= last; + !last + }) + .map(|data| serde_json::from_str(data).unwrap()) + .collect(); + (frames, done) +} + +/// The tool calls the chunks stream, joined: (id, name, arguments) per index. +fn streamed_tool_calls(chunks: &[Value]) -> Vec<(String, String, String)> { + let mut calls: Vec<(String, String, String)> = Vec::new(); + for chunk in chunks { + let Some(deltas) = chunk["choices"][0]["delta"]["tool_calls"].as_array() else { + continue; + }; + for delta in deltas { + let index = delta["index"].as_u64().unwrap() as usize; + if calls.len() <= index { + calls.resize(index + 1, Default::default()); + } + let call = &mut calls[index]; + if let Some(id) = delta["id"].as_str() { + call.0 = id.to_owned(); + } + if let Some(name) = delta["function"]["name"].as_str() { + call.1 = name.to_owned(); + } + call.2 + .push_str(delta["function"]["arguments"].as_str().unwrap_or_default()); + } + } + calls +} + +fn finish_reason(chunks: &[Value]) -> Option { + chunks + .iter() + .find_map(|chunk| chunk["choices"][0]["finish_reason"].as_str()) + .map(str::to_owned) +} + +#[tokio::test] +async fn anthropic_tool_round_trip() { + let harness = harness().await; + + let asked = harness + .json(json!({ + "model": "claude", + "messages": [question()], + "tools": [weather_tool()], + "tool_choice": "required", + "max_tokens": 64, + })) + .await; + let message = &asked["choices"][0]["message"]; + assert_eq!(asked["choices"][0]["finish_reason"], "tool_calls"); + assert_eq!(message["content"], "Checking."); + assert_eq!( + message["tool_calls"], + json!([{"id": "toolu_1", "type": "function", "index": 1, + "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}}]) + ); + + let done = harness + .json(json!({ + "model": "claude", + "messages": answered("toolu_1"), + "tools": [weather_tool()], + "max_tokens": 64, + })) + .await; + assert_eq!(done["choices"][0]["message"]["content"], "Sunny."); + assert_eq!(done["choices"][0]["finish_reason"], "stop"); + + let sent = harness.sent(); + assert_eq!(sent[0].path, "/v1/messages"); + assert_eq!( + sent[0].body, + json!({ + "model": ANTHROPIC_MODEL, + "messages": [{"role": "user", "content": [{"type": "text", "text": "Weather in Paris?"}]}], + "max_tokens": 64, + "tools": [{"name": "get_weather", "type": "custom", "description": "The weather in a city", + "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}], + "tool_choice": {"type": "any"} + }) + ); + assert_eq!( + sent[1].body["messages"], + json!([ + {"role": "user", "content": [{"type": "text", "text": "Weather in Paris?"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}}]}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "toolu_1", "content": "sunny, 21C"}]} + ]) + ); +} + +#[tokio::test] +async fn bedrock_tool_round_trip() { + let harness = harness().await; + + let asked = harness + .json(json!({ + "model": "bedrock", + "messages": [question()], + "tools": [weather_tool()], + "tool_choice": {"type": "function", "function": {"name": "get_weather"}}, + "max_tokens": 64, + })) + .await; + assert_eq!(asked["choices"][0]["finish_reason"], "tool_calls"); + assert_eq!( + asked["choices"][0]["message"]["tool_calls"], + json!([{"id": "tooluse_1", "type": "function", "index": 1, + "function": {"name": "get_weather", "arguments": "{\"city\": \"Paris\"}"}}]) + ); + + let done = harness + .json(json!({ + "model": "bedrock", + "messages": answered("tooluse_1"), + "tools": [weather_tool()], + "max_tokens": 64, + })) + .await; + assert_eq!(done["choices"][0]["message"]["content"], "Sunny."); + assert_eq!(done["choices"][0]["finish_reason"], "stop"); + + let sent = harness.sent(); + assert_eq!(sent[0].path, format!("/model/{BEDROCK_MODEL}/converse")); + assert_eq!( + sent[0].body["toolConfig"], + json!({ + "tools": [{"toolSpec": {"name": "get_weather", "description": "The weather in a city", + "inputSchema": {"json": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}}], + "toolChoice": {"tool": {"name": "get_weather"}} + }) + ); + assert_eq!( + sent[1].body["messages"], + json!([ + {"role": "user", "content": [{"text": "Weather in Paris?"}]}, + {"role": "assistant", "content": [{"toolUse": {"toolUseId": "tooluse_1", "name": "get_weather", "input": {"city": "Paris"}}}]}, + {"role": "user", "content": [{"toolResult": {"toolUseId": "tooluse_1", "content": [{"text": "sunny, 21C"}]}}]} + ]) + ); +} + +#[rstest] +#[case::anthropic("claude", "toolu_1")] +#[case::bedrock("bedrock", "tooluse_1")] +#[tokio::test] +async fn a_streamed_tool_call_arrives_as_openai_tool_call_chunks( + #[case] model: &str, + #[case] id: &str, +) { + let harness = harness().await; + + let (status, body) = harness + .send(json!({ + "model": model, + "messages": [question()], + "tools": [weather_tool()], + "max_tokens": 64, + "stream": true, + })) + .await; + + assert_eq!(status, 200, "{body}"); + let (chunks, done) = frames(&body); + assert!(done, "{body}"); + assert_eq!( + streamed_tool_calls(&chunks), + [( + id.to_owned(), + "get_weather".to_owned(), + "{\"city\": \"Paris\"}".to_owned() + )] + ); + assert_eq!(finish_reason(&chunks).as_deref(), Some("tool_calls")); + let sent = &harness.sent()[0]; + let tools = match model { + "claude" => sent.body["tools"][0]["name"].clone(), + _ => sent.body["toolConfig"]["tools"][0]["toolSpec"]["name"].clone(), + }; + assert_eq!(tools, "get_weather"); +} + +#[rstest] +#[case::max_tokens("max_tokens")] +#[case::max_completion_tokens("max_completion_tokens")] +#[tokio::test] +async fn bedrock_takes_the_token_limit_in_its_inference_config(#[case] param: &str) { + let harness = harness().await; + + harness + .json(json!({ + "model": "bedrock", + "messages": [{"role": "system", "content": "Be brief."}, question()], + param: 32, + "temperature": 0.5, + "stop": ["END"], + })) + .await; + + let sent = &harness.sent()[0]; + assert_eq!( + sent.body["inferenceConfig"], + json!({"maxTokens": 32, "temperature": 0.5, "stopSequences": ["END"]}) + ); + assert_eq!(sent.body["system"], json!([{"text": "Be brief."}])); +} + +#[tokio::test] +async fn anthropic_gets_the_model_default_max_tokens_when_the_caller_names_none() { + let harness = harness().await; + + harness + .json(json!({"model": "claude", "messages": [question()]})) + .await; + + assert_eq!(harness.sent()[0].body["max_tokens"], 8192); +} + +#[tokio::test] +async fn an_image_data_url_reaches_each_provider_inline() { + let harness = harness().await; + let messages = json!([{"role": "user", "content": [ + {"type": "text", "text": "What is this?"}, + {"type": "image_url", "image_url": {"url": format!("data:image/png;base64,{PNG}")}} + ]}]); + + harness + .json(json!({"model": "claude", "messages": messages, "max_tokens": 16})) + .await; + harness + .json(json!({"model": "bedrock", "messages": messages, "max_tokens": 16})) + .await; + + let sent = harness.sent(); + assert_eq!( + sent[0].body["messages"][0]["content"][1], + json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": PNG}}) + ); + assert_eq!( + sent[1].body["messages"][0]["content"][1], + json!({"image": {"format": "png", "source": {"bytes": PNG}}}) + ); +} + +#[tokio::test] +async fn anthropic_response_format_comes_back_as_the_answer() { + let harness = harness().await; + let schema = + json!({"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}); + + let answer = harness + .json(json!({ + "model": "claude", + "messages": [question()], + "max_tokens": 64, + "response_format": {"type": "json_schema", "json_schema": {"name": "place", "schema": schema}}, + "user": "end-user-1", + })) + .await; + + assert_eq!( + answer["choices"][0]["message"]["content"], + "{\"city\": \"Paris\"}" + ); + assert_eq!(answer["choices"][0]["message"]["tool_calls"], Value::Null); + assert_eq!(answer["choices"][0]["finish_reason"], "stop"); + let sent = &harness.sent()[0]; + assert_eq!(sent.body["metadata"], json!({"user_id": "end-user-1"})); + assert_eq!(sent.body["tools"][0]["name"], "json_tool_call"); + assert_eq!(sent.body["tools"][0]["input_schema"], schema); + assert_eq!( + sent.body["tool_choice"], + json!({"name": "json_tool_call", "type": "tool"}) + ); +} + +#[rstest] +#[case::anthropic("claude", json!({"logprobs": true}))] +#[case::bedrock("bedrock", json!({"tool_choice": "none", "tools": [weather_tool()]}))] +#[case::n_choices("claude", json!({"n": 2}))] +#[tokio::test] +async fn an_unsupported_param_is_refused_and_dropped_with_drop_params( + #[case] model: &str, + #[case] extra: Value, +) { + let harness = harness().await; + let mut body = json!({"model": model, "messages": [question()], "max_tokens": 16}); + body.as_object_mut() + .unwrap() + .extend(extra.as_object().unwrap().clone()); + + let (status, refused) = harness.send(body.clone()).await; + assert_eq!(status, 400, "{refused}"); + assert!( + harness.sent().is_empty(), + "a refused request never goes out" + ); + + body["drop_params"] = json!(true); + harness.json(body).await; + let sent = &harness.sent()[0]; + for name in extra.as_object().unwrap().keys() { + if name == "tools" { + continue; + } + assert!( + sent.body.get(name).is_none(), + "{name} reached {}", + sent.body + ); + } + assert!( + sent.body + .get("toolConfig") + .and_then(|config| config.get("toolChoice")) + .is_none() + ); + assert!(sent.body.get("drop_params").is_none()); +} + +#[tokio::test] +async fn a_deployment_that_drops_params_drops_them_without_the_request_asking() { + let harness = harness().await; + + harness + .json(json!({ + "model": "claude-dropping", + "messages": [question()], + "max_tokens": 16, + "logprobs": true, + })) + .await; + + assert!(harness.sent()[0].body.get("logprobs").is_none()); +} + +#[tokio::test] +async fn a_cache_control_part_reaches_anthropic_and_bedrock() { + let harness = harness().await; + let messages = json!([ + {"role": "system", "content": [{"type": "text", "text": "Long instructions.", "cache_control": {"type": "ephemeral"}}]}, + question() + ]); + + harness + .json(json!({"model": "claude", "messages": messages, "max_tokens": 16})) + .await; + harness + .json(json!({"model": "bedrock", "messages": messages, "max_tokens": 16})) + .await; + + let sent = harness.sent(); + assert_eq!( + sent[0].body["system"], + json!([{"type": "text", "text": "Long instructions.", "cache_control": {"type": "ephemeral"}}]) + ); + assert_eq!( + sent[1].body["system"], + json!([{"text": "Long instructions."}, {"cachePoint": {"type": "default"}}]) + ); + assert!(sent[0].header("x-api-key").is_some()); +} diff --git a/litellm-rust/crates/inference-chat/Cargo.toml b/litellm-rust/crates/inference-chat/Cargo.toml index 4cc32bb7656..aa2ca0f5e8a 100644 --- a/litellm-rust/crates/inference-chat/Cargo.toml +++ b/litellm-rust/crates/inference-chat/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true repository.workspace = true [dependencies] +base64.workspace = true bytes.workspace = true futures-util.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } diff --git a/litellm-rust/crates/inference-chat/src/handler.rs b/litellm-rust/crates/inference-chat/src/handler.rs index 5f4d00e1d7c..95764ccda42 100644 --- a/litellm-rust/crates/inference-chat/src/handler.rs +++ b/litellm-rust/crates/inference-chat/src/handler.rs @@ -29,7 +29,7 @@ pub(super) async fn execute( config, url, body, - stream_shape: _, + stream_shape, optional_params, environment, secrets, @@ -122,7 +122,7 @@ pub(super) async fn execute( )) })?; config - .transform_response(&model, ProviderChatResponseData { body }) + .transform_response(&model, ProviderChatResponseData { body }, &stream_shape) .map_err(Error::from) .map_err(as_response_error) }, @@ -229,6 +229,8 @@ mod tests { custom_llm_provider: None, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, }) .unwrap(), std::sync::Arc::new(|_: &str| None), diff --git a/litellm-rust/crates/inference-chat/src/lib.rs b/litellm-rust/crates/inference-chat/src/lib.rs index db4f28f4d2a..df442ea2489 100644 --- a/litellm-rust/crates/inference-chat/src/lib.rs +++ b/litellm-rust/crates/inference-chat/src/lib.rs @@ -8,6 +8,7 @@ pub use litellm_inference::RouteError as Error; mod common_utils; pub mod constants; pub(crate) mod handler; +mod media; pub mod openai; mod prepare; pub mod stream; @@ -25,6 +26,7 @@ pub struct ChatCompletionsRoute { auth: Arc, secrets: Arc, cache: Option, + media: Option, } impl ChatCompletionsRoute { @@ -38,6 +40,16 @@ impl ChatCompletionsRoute { auth, secrets, cache: None, + media: None, + } + } + + /// Downloads the media URLs a provider cannot take by reference. Without one such a + /// request is refused as unsupported. + pub fn with_media(self, media: litellm_http::media::MediaFetcher) -> Self { + Self { + media: Some(media), + ..self } } @@ -77,7 +89,8 @@ impl ChatCompletionsRoute { interceptors: &impl litellm_host::interceptors::Interceptors, observers: Option<&ObservationSender>, ) -> Result { - let resolved = resolve_request(request)?; + let mut resolved = resolve_request(request)?; + self.inline_remote_media(&mut resolved).await?; let snapshot = self .secrets .resolve(&resolved.config.secret_names()) diff --git a/litellm-rust/crates/inference-chat/src/media.rs b/litellm-rust/crates/inference-chat/src/media.rs new file mode 100644 index 00000000000..8287b496eba --- /dev/null +++ b/litellm-rust/crates/inference-chat/src/media.rs @@ -0,0 +1,158 @@ +//! The media a translated request needs downloaded first: an image or file URL the provider +//! cannot take by reference, which Python fetches while it builds the request. + +use std::{collections::HashMap, time::Duration}; + +use base64::Engine; +use futures_util::future::try_join_all; +use litellm_http::media::DownloadPolicy; +use litellm_llms::base_llm::chat::media::{ + MAX_MEDIA_DOWNLOAD_BYTES, downloaded_media_type, inline_media, +}; + +use crate::{ChatCompletionsRoute, Error, types::ResolvedChatCompletionsRequest}; + +const MEDIA_DOWNLOAD: DownloadPolicy = DownloadPolicy { + timeout: Duration::from_secs(60), + max_bytes: MAX_MEDIA_DOWNLOAD_BYTES, + max_redirects: 10, +}; + +impl ChatCompletionsRoute { + /// Downloads every URL the config names and inlines each as a data URL. A route with no + /// fetcher leaves such a request to the host; a URL that cannot be fetched is the + /// caller's error, as Python's image fetch failure is. + pub(crate) async fn inline_remote_media( + &self, + request: &mut ResolvedChatCompletionsRequest<'_>, + ) -> Result<(), Error> { + let urls = request.config.remote_media(&request.messages); + if urls.is_empty() { + return Ok(()); + } + let fetcher = self + .media + .as_ref() + .ok_or(Error::Unsupported("message media URL"))?; + let fetch = |url: String| async move { + let unable = |reason: String| { + Error::InvalidRequest( + format!("Unable to fetch media from url {url}: {reason}").into(), + ) + }; + let parsed = reqwest::Url::parse(&url).map_err(|err| unable(err.to_string()))?; + let media = fetcher + .fetch(parsed, MEDIA_DOWNLOAD) + .await + .map_err(|err| unable(err.to_string()))?; + let data = format!( + "data:{};base64,{}", + downloaded_media_type(&url, &media.content_type), + base64::engine::general_purpose::STANDARD.encode(media.bytes) + ); + Ok::<_, Error>((url, data)) + }; + let downloaded: HashMap = try_join_all(urls.into_iter().map(fetch)) + .await? + .into_iter() + .collect(); + inline_media(&mut request.messages, &downloaded); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use litellm_http::{Client, media::MediaFetcher}; + + use rstest::rstest; + use serde_json::{Value, json}; + use wiremock::{ + Mock, MockServer, ResponseTemplate, + matchers::{method, path}, + }; + + use crate::{ + ChatCompletionsRoute, Error, prepare::resolve_request, types::ChatCompletionsRequest, + }; + + fn route(media: bool) -> ChatCompletionsRoute { + let route = crate::test_support::route(); + match media { + true => route.with_media(MediaFetcher::for_test(Client::plain_for_test())), + false => route, + } + } + + fn request(url: &str) -> ChatCompletionsRequest<'static> { + ChatCompletionsRequest { + model: "anthropic/claude-sonnet-4-5", + messages: json!([{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": url}} + ]}]), + optional_params: serde_json::Map::new(), + api_key: Some("sk-test"), + api_base: None, + custom_llm_provider: None, + extra_headers: None, + timeout: None, + drop_params: Some(false), + model_info: None, + } + } + + fn image(request: &crate::types::ResolvedChatCompletionsRequest<'_>) -> Value { + serde_json::to_value(&request.messages[0].content).unwrap()[0]["image_url"]["url"].clone() + } + + #[rstest] + #[tokio::test] + async fn a_remote_image_is_downloaded_and_inlined() { + let server = MockServer::start().await; + Mock::given(method("GET")) + .and(path("/cat.png")) + .respond_with(ResponseTemplate::new(200).set_body_raw(b"png".to_vec(), "image/png")) + .mount(&server) + .await; + let url = format!("{}/cat.png", server.uri()); + let url: &'static str = Box::leak(url.into_boxed_str()); + let mut resolved = resolve_request(request(url)).unwrap(); + + route(true) + .inline_remote_media(&mut resolved) + .await + .unwrap(); + + assert_eq!(image(&resolved), "data:image/png;base64,cG5n"); + } + + #[rstest] + #[tokio::test] + async fn a_media_url_that_cannot_be_fetched_is_the_callers_error() { + let server = MockServer::start().await; + let url = format!("{}/missing.png", server.uri()); + let url: &'static str = Box::leak(url.into_boxed_str()); + let mut resolved = resolve_request(request(url)).unwrap(); + + let error = route(true) + .inline_remote_media(&mut resolved) + .await + .unwrap_err(); + + assert!(matches!(error, Error::InvalidRequest(_)), "{error:?}"); + } + + #[rstest] + #[tokio::test] + async fn without_a_fetcher_a_media_url_is_unsupported_and_a_data_url_is_left_alone() { + let mut remote = resolve_request(request("http://example.com/cat.png")).unwrap(); + assert!(matches!( + route(false).inline_remote_media(&mut remote).await, + Err(Error::Unsupported(_)) + )); + + let mut inline = resolve_request(request("data:image/png;base64,cG5n")).unwrap(); + route(false).inline_remote_media(&mut inline).await.unwrap(); + assert_eq!(image(&inline), "data:image/png;base64,cG5n"); + } +} diff --git a/litellm-rust/crates/inference-chat/src/prepare.rs b/litellm-rust/crates/inference-chat/src/prepare.rs index 058ca245e79..5db550efd01 100644 --- a/litellm-rust/crates/inference-chat/src/prepare.rs +++ b/litellm-rust/crates/inference-chat/src/prepare.rs @@ -1,7 +1,10 @@ use litellm_auth::SecretValue; use litellm_core_utils::settings::Lookup; use litellm_http::request::with_default_headers; -use litellm_llms::base_llm::{auth::ValidatedEnvironment, chat::transformation::BaseConfig}; +use litellm_llms::base_llm::{ + auth::{Headers, ValidatedEnvironment}, + chat::{params::ChatRequestContext, transformation::BaseConfig}, +}; use litellm_llms_types::formats::chat_completions::ChatMessage; use litellm_secrets::source::Secrets; use serde_json::Value; @@ -72,9 +75,38 @@ pub(super) fn resolve_request( api_base: request.api_base, extra_headers: request.extra_headers, timeout: request.timeout, + drop_params: request.drop_params, + model_info: request.model_info, }) } +/// The headers a transformation asked for, added to the ones already going out. A header +/// already there (a forwarded `anthropic-beta`, say) keeps its values and gains the new ones, +/// comma separated, the way Python unions beta headers. +fn merge_headers(headers: &mut Headers, added: Headers) { + for (name, value) in added { + match headers + .iter_mut() + .find(|(existing, _)| existing.eq_ignore_ascii_case(&name)) + { + Some((_, existing)) => { + let mut values: Vec<&str> = existing + .split(',') + .map(str::trim) + .filter(|value| !value.is_empty()) + .collect(); + for value in value.split(',').map(str::trim) { + if !value.is_empty() && !values.contains(&value) { + values.push(value); + } + } + *existing = values.join(", "); + } + None => headers.push((name, value)), + } + } +} + fn validate_environment( request: &ResolvedChatCompletionsRequest<'_>, model: &str, @@ -101,7 +133,7 @@ pub(super) fn prepare_provider_request( request: ResolvedChatCompletionsRequest<'_>, secrets: Secrets, ) -> Result { - let environment = + let mut environment = validate_environment(&request, &request.model, request.config, secrets.as_ref())?; let model = request.model; let config = request.config; @@ -112,8 +144,17 @@ pub(super) fn prepare_provider_request( &request.optional_params, &env_lookup, )?; - let transformed = - config.transform_request(&model, request.messages, request.optional_params.clone())?; + let context = ChatRequestContext { + drop_params: request.drop_params, + model_info: request.model_info.as_deref(), + }; + let transformed = config.transform_request( + &model, + request.messages, + request.optional_params.clone(), + &context, + )?; + merge_headers(&mut environment.headers, transformed.headers); Ok(ProviderChatCompletionsRequest { model, @@ -136,7 +177,7 @@ mod tests { use litellm_llms::base_llm::auth::{AuthScheme, resolve_auth}; use serde_json::{Map, Value, json}; - use super::{prepare_provider_request, resolve_request}; + use super::{merge_headers, prepare_provider_request, resolve_request}; use crate::{ Error, types::{ChatCompletionsRequest, ProviderChatCompletionsRequest}, @@ -183,6 +224,8 @@ mod tests { custom_llm_provider: provider, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, } } @@ -636,4 +679,27 @@ mod tests { "prepare did not carry the bearer token" ); } + + #[test] + fn transformation_headers_union_with_the_forwarded_ones() { + let mut headers = vec![ + ("Anthropic-Beta".to_owned(), "a,b".to_owned()), + ("x-other".to_owned(), "1".to_owned()), + ]; + merge_headers( + &mut headers, + vec![ + ("anthropic-beta".to_owned(), "b, c".to_owned()), + ("x-new".to_owned(), "2".to_owned()), + ], + ); + assert_eq!( + headers, + [ + ("Anthropic-Beta".to_owned(), "a, b, c".to_owned()), + ("x-other".to_owned(), "1".to_owned()), + ("x-new".to_owned(), "2".to_owned()), + ] + ); + } } diff --git a/litellm-rust/crates/inference-chat/src/route.rs b/litellm-rust/crates/inference-chat/src/route.rs index 50fc412a705..73eb69ddc0c 100644 --- a/litellm-rust/crates/inference-chat/src/route.rs +++ b/litellm-rust/crates/inference-chat/src/route.rs @@ -69,6 +69,8 @@ impl ChatCompletionsRoute { custom_llm_provider: call.custom_llm_provider.as_deref(), extra_headers: call.extra_headers, timeout: call.timeout, + drop_params: call.drop_params, + model_info: call.model_info, }; self.run(request, cache_options, interceptors, observers) .await diff --git a/litellm-rust/crates/inference-chat/src/stream.rs b/litellm-rust/crates/inference-chat/src/stream.rs index 33dd1d8a441..c7d15e9eb0a 100644 --- a/litellm-rust/crates/inference-chat/src/stream.rs +++ b/litellm-rust/crates/inference-chat/src/stream.rs @@ -140,7 +140,7 @@ impl ChatCompletionsRoute { let timeout = call .timeout .unwrap_or(Duration::from_secs(CHAT_COMPLETIONS_TIMEOUT_SECS)); - let resolved = resolve_request(ChatCompletionsRequest { + let mut resolved = resolve_request(ChatCompletionsRequest { model: &call.model, messages: call.messages, optional_params, @@ -149,7 +149,10 @@ impl ChatCompletionsRoute { custom_llm_provider: call.custom_llm_provider.as_deref(), extra_headers: call.extra_headers, timeout: call.timeout, + drop_params: call.drop_params, + model_info: call.model_info, })?; + self.inline_remote_media(&mut resolved).await?; let snapshot = self .secrets .resolve(&resolved.config.secret_names()) @@ -1019,6 +1022,8 @@ mod tests { custom_llm_provider: None, extra_headers: None, timeout: Some(Duration::from_secs(5)), + drop_params: Some(false), + model_info: None, }, response_model: "public/model".into(), usage: ReportedUsage::default(), diff --git a/litellm-rust/crates/inference-chat/src/types.rs b/litellm-rust/crates/inference-chat/src/types.rs index d0cb7b24d6c..e02b67e5d2b 100644 --- a/litellm-rust/crates/inference-chat/src/types.rs +++ b/litellm-rust/crates/inference-chat/src/types.rs @@ -1,20 +1,20 @@ use litellm_secrets::source::Secrets; -use std::time::Duration; +use std::{sync::Arc, time::Duration}; use litellm_auth::SecretValue; use litellm_llms::base_llm::{ auth::ValidatedEnvironment, chat::{streaming::StreamShape, transformation::BaseConfig}, + model_info::ModelInfo, }; use litellm_llms_types::formats::chat_completions::ChatMessage; use serde_json::{Map, Value}; /// A `/chat/completions` call as it crosses into the core. /// -/// `optional_params` arrives already mapped to the provider's own parameter -/// names by the host, exactly as the messages route receives an already -/// Anthropic-shaped body. The core owns the conversation translation, the -/// provider call, and the response normalization. +/// `optional_params` are the OpenAI request's own params; the provider config maps them, +/// refusing what the provider cannot take unless `drop_params` says to drop it. The core +/// owns the conversation translation, the provider call, and the response normalization. pub struct ChatCompletionsRequest<'a> { pub model: &'a str, pub messages: Value, @@ -24,6 +24,11 @@ pub struct ChatCompletionsRequest<'a> { pub custom_llm_provider: Option<&'a str>, pub extra_headers: Option>, pub timeout: Option, + /// `drop_params` from the deployment or the proxy settings, `None` when neither says. A + /// request's own `drop_params` rides in `optional_params`. + pub drop_params: Option, + /// The model cost map, for the param rules that read model capabilities. + pub model_info: Option>, } pub struct ChatCompletionsCall { @@ -35,6 +40,11 @@ pub struct ChatCompletionsCall { pub custom_llm_provider: Option, pub extra_headers: Option>, pub timeout: Option, + /// `drop_params` from the deployment or the proxy settings, `None` when neither says. A + /// request's own `drop_params` rides in `optional_params`. + pub drop_params: Option, + /// The model cost map, for the param rules that read model capabilities. + pub model_info: Option>, } impl From> for ChatCompletionsCall { @@ -48,6 +58,8 @@ impl From> for ChatCompletionsCall { custom_llm_provider: request.custom_llm_provider.map(str::to_owned), extra_headers: request.extra_headers, timeout: request.timeout, + drop_params: request.drop_params, + model_info: request.model_info, } } } @@ -62,6 +74,11 @@ pub struct ResolvedChatCompletionsRequest<'a> { pub api_base: Option<&'a str>, pub extra_headers: Option>, pub timeout: Option, + /// `drop_params` from the deployment or the proxy settings, `None` when neither says. A + /// request's own `drop_params` rides in `optional_params`. + pub drop_params: Option, + /// The model cost map, for the param rules that read model capabilities. + pub model_info: Option>, } pub struct ProviderChatCompletionsRequest { diff --git a/litellm-rust/crates/inference-chat/tests/caching.rs b/litellm-rust/crates/inference-chat/tests/caching.rs index 3fc432f8b46..69b6ed917be 100644 --- a/litellm-rust/crates/inference-chat/tests/caching.rs +++ b/litellm-rust/crates/inference-chat/tests/caching.rs @@ -84,6 +84,8 @@ async fn the_same_route_entrypoint_reports_facts_with_or_without_caching( custom_llm_provider: None, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, }, &(), Some(observer.clone()), @@ -254,6 +256,8 @@ async fn chat_cache_identity_follows_resolved_configuration_and_request_callback custom_llm_provider: None, extra_headers: None, timeout: None, + drop_params: Some(false), + model_info: None, }, &hooks, None, @@ -319,6 +323,8 @@ async fn signed_requests_bypass_response_caching(cache: Arc ChatCompletionsRequest<'static> { custom_llm_provider: None, extra_headers: None, timeout: Some(Duration::from_secs(10)), + drop_params: Some(false), + model_info: None, } } @@ -163,7 +165,7 @@ async fn bedrock_round_trip_is_signed_and_normalized(request: ChatCompletionsReq #[case::missing_usage( r#"{"model":"m","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn"}"# )] -#[case::tool_use_block(r#"{"model":"m","content":[{"type":"tool_use","id":"t","name":"f","input":{}}],"stop_reason":"tool_use","usage":{"input_tokens":1,"output_tokens":1}}"#)] +#[case::malformed_token_count(r#"{"model":"m","content":[{"type":"text","text":"hi"}],"stop_reason":"end_turn","usage":{"input_tokens":"1","output_tokens":1}}"#)] #[case::not_json("not json")] #[tokio::test] async fn a_response_it_cannot_normalize_is_reported_as_already_sent( @@ -294,6 +296,8 @@ async fn direct_and_hosted_calls_share_hooks_and_lifecycle( custom_llm_provider: call.custom_llm_provider.as_deref(), extra_headers: call.extra_headers, timeout: call.timeout, + drop_params: Some(false), + model_info: None, }, &host, Some(host.events.0.sender.clone()), diff --git a/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs b/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs index 44d66d5ebe6..5166d8b5ea6 100644 --- a/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs +++ b/litellm-rust/crates/llms-types/src/formats/chat_completions/response.rs @@ -87,6 +87,7 @@ pub struct ChatCompletionsUsage { } #[macro_rules_attribute::apply(crate::wire_type)] +#[derive(Default)] pub struct ChatCompletionsChoiceMessage { pub role: String, // Whether an empty turn is `None` or `""` is the provider's choice, not a @@ -94,6 +95,14 @@ pub struct ChatCompletionsChoiceMessage { // while Converse assigns the joined string unconditionally. Each config // mirrors its own, so keep this optional and serialize it even when None. pub content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub tool_calls: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reasoning_content: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub thinking_blocks: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider_specific_fields: Option>, } #[macro_rules_attribute::apply(crate::wire_type)] diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 2d3ea85d1fc..629b6ee75e1 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -22,6 +22,8 @@ litellm-secrets.workspace = true litellm-python-compat.workspace = true litellm-router-types.workspace = true base64.workspace = true +sha2.workspace = true +uuid.workspace = true bytes.workspace = true data-url = "0.3.2" futures-util.workspace = true diff --git a/litellm-rust/crates/llms/src/anthropic/chat/handler.rs b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs index a373e22873a..27eb184ed24 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/handler.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/handler.rs @@ -4,8 +4,9 @@ //! Each event becomes one provider chunk, empty ones included; the host's stream wrapper //! (Python's `CustomStreamWrapper`) decides which of them the client sees. //! -//! DIVERGENCE: not built here, since the chat route never asks for what produces them: -//! `json_mode` / `response_format` tool conversion, `web_search_calls` and +//! DIVERGENCE: a `response_format` tool's answer is held until its block ends and sent as +//! one piece of content, where Python streams it as it decides the envelope; an answer +//! spread over several blocks is sent per block. Not built: `web_search_calls` and //! `code_interpreter_results` summaries, and a tool call's `caller`. The reasoning token //! estimate Python makes from streamed thinking text (with its tokenizer) is not made, so //! such a stream reports `completion_tokens_details` without counts unless Anthropic sends @@ -28,7 +29,14 @@ use serde_json::{Map, Value, json}; use crate::{ Error, - base_llm::{base_model_iterator::StreamTransformer, chat::streaming::StreamShape}, + base_llm::{ + base_model_iterator::StreamTransformer, + chat::{ + params::{RESPONSE_FORMAT_TOOL_NAME, python_dumps}, + response_format::{envelope_contents, json_tool_answer}, + streaming::StreamShape, + }, + }, }; /// Anthropic sends no status with an `error` event; Python raises it as a 500. @@ -51,6 +59,10 @@ pub struct ModelResponseIterator { web_search_results: Vec, tool_results: Vec, compaction_blocks: Vec, + /// Whether the tool block in progress is the `response_format` tool. + is_response_format_tool: bool, + converted_response_format_tool: bool, + response_format_buffer: String, } /// What one event contributes to its chunk. @@ -79,9 +91,58 @@ impl ModelResponseIterator { web_search_results: Vec::new(), tool_results: Vec::new(), compaction_blocks: Vec::new(), + is_response_format_tool: false, + converted_response_format_tool: false, + response_format_buffer: String::new(), } } + /// `_handle_json_mode_chunk`: the `response_format` tool's arguments become content. + fn handle_json_mode(&mut self, parsed: &mut Parsed) { + if !self.shape.json_mode { + return; + } + let Some(tool_use) = parsed.tool_use.take() else { + return; + }; + if tool_use.id.is_some() { + self.is_response_format_tool = + tool_use.function.name.as_deref() == Some(RESPONSE_FORMAT_TOOL_NAME); + if self.is_response_format_tool { + self.tool_index -= 1; + } + } + if !self.is_response_format_tool { + parsed.tool_use = Some(tool_use); + return; + } + self.converted_response_format_tool = true; + if self.shape.response_format_schema.is_some() { + match tool_use.id { + Some(_) => self.response_format_buffer.clear(), + None => self + .response_format_buffer + .push_str(&tool_use.function.arguments), + } + parsed.text = String::new(); + return; + } + parsed.text = json_tool_answer(&[tool_use.function.arguments], None); + } + + /// `_held_answer`: the held arguments, or what their envelope wraps. + fn flush_response_format_buffer(&mut self) -> String { + let held = std::mem::take(&mut self.response_format_buffer); + let Some(schema) = &self.shape.response_format_schema else { + return String::new(); + }; + serde_json::from_str::(&held) + .ok() + .and_then(|payload| envelope_contents(&payload, schema, RESPONSE_FORMAT_TOOL_NAME)) + .map(|contents| python_dumps(&contents)) + .unwrap_or(held) + } + /// Whether the tool block so far streamed only empty argument fragments. pub fn check_empty_tool_call_args(&self) -> bool { match self.content_blocks.first() { @@ -273,6 +334,10 @@ impl ModelResponseIterator { if in_tool && self.check_empty_tool_call_args() { parsed.tool_use = Some(self.tool_call(None, None, "{}".into())); } + if self.is_response_format_tool { + parsed.text = self.flush_response_format_buffer(); + } + self.is_response_format_tool = false; self.current_content_block_type = None; parsed } @@ -282,10 +347,16 @@ impl ModelResponseIterator { delta: MessagesDelta, usage: Option, ) -> Parsed { + let provider_finish_reason = + finish_reason_for(delta.stop_reason.as_deref().unwrap_or("stop")).to_string(); + let called_only_the_response_format_tool = provider_finish_reason == "tool_calls" + && self.converted_response_format_tool + && self.tool_index < 0; let mut parsed = Parsed { - finish_reason: Some( - finish_reason_for(delta.stop_reason.as_deref().unwrap_or("stop")).to_string(), - ), + finish_reason: Some(match called_only_the_response_format_tool { + true => "stop".to_owned(), + false => provider_finish_reason, + }), usage: usage.map(|usage| self.handle_usage(&usage)), ..Parsed::default() }; @@ -314,7 +385,7 @@ impl ModelResponseIterator { &mut self, event: MessagesStreamEvent, ) -> Result { - let parsed = match event { + let mut parsed = match event { MessagesStreamEvent::ContentBlockDelta { delta, .. } => self.content_block_delta(delta), MessagesStreamEvent::ContentBlockStart { content_block, .. } => { self.content_block_start(content_block) @@ -341,6 +412,7 @@ impl ModelResponseIterator { }); } }; + self.handle_json_mode(&mut parsed); Ok(self.chunk(parsed)) } diff --git a/litellm-rust/crates/llms/src/anthropic/chat/messages.rs b/litellm-rust/crates/llms/src/anthropic/chat/messages.rs new file mode 100644 index 00000000000..8f4559f1e73 --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/chat/messages.rs @@ -0,0 +1,777 @@ +//! Chat messages as the Messages API takes them: Python's `translate_system_message`, +//! `place_mid_conversation_system` and `anthropic_messages_pt`. + +use std::collections::HashSet; + +use base64::Engine; +use serde_json::{Map, Value, json}; + +use crate::{ + Error, ErrorDetail, + anthropic::common_utils::ENCRYPTED_REASONING_SIGNATURE_PREFIX, + base_llm::chat::{ + media::{data_url, image_url}, + params::{parse_tool_arguments, truthy}, + }, +}; + +pub const EMPTY_TEXT_PLACEHOLDER: &str = + "[System: Empty message content sanitised to satisfy protocol]"; +pub const CONVERTED_SYSTEM_NOTE: &str = "Operator note (not from the user): the following was originally a mid-conversation system-role reminder."; +const TOOL_USE_ID_FALLBACK: &str = "tool_use_id"; + +fn bad_request(message: impl Into) -> Error { + Error::InvalidRequest(ErrorDetail::Message(message.into())) +} + +pub fn role(message: &Value) -> &str { + match message.get("role").and_then(Value::as_str) { + Some("developer") => "system", + Some(role) => role, + None => "", + } +} + +/// Python's truthiness of a message's `content`. +fn content_is_empty(message: &Value) -> bool { + match message.get("content") { + None | Some(Value::Null) => true, + Some(Value::String(text)) => text.is_empty(), + Some(Value::Array(items)) => items.is_empty(), + Some(_) => false, + } +} + +/// `[^a-zA-Z0-9_-]` -> `_`, an empty id becoming `tool_use_id`. +pub fn sanitize_tool_use_id(id: &str) -> String { + let sanitized: String = id + .chars() + .map( + |c| match c.is_ascii_alphanumeric() || c == '_' || c == '-' { + true => c, + false => '_', + }, + ) + .collect(); + match sanitized.is_empty() { + true => TOOL_USE_ID_FALLBACK.to_owned(), + false => sanitized, + } +} + +/// `add_cache_control_to_content`: a dict `cache_control` on the source is copied over. +fn with_cache_control(mut block: Value, source: &Value) -> Value { + if let (Some(cache_control @ Value::Object(_)), Value::Object(fields)) = + (source.get("cache_control"), &mut block) + { + fields.insert("cache_control".into(), cache_control.clone()); + } + block +} + +/// `_cache_control` of the mid-conversation module: only an ephemeral one, rebuilt. +fn normalized_cache_control(source: &Value) -> Option { + let cache_control = source.get("cache_control")?.as_object()?; + if cache_control.get("type").and_then(Value::as_str) != Some("ephemeral") { + return None; + } + Some(match cache_control.get("ttl").and_then(Value::as_str) { + Some(ttl @ ("1h" | "5m")) => json!({"type": "ephemeral", "ttl": ttl}), + _ => json!({"type": "ephemeral"}), + }) +} + +/// The leading run of system messages, and the rest. +pub fn split_leading_system_run(messages: Vec) -> (Vec, Vec) { + let leading = messages + .iter() + .position(|message| role(message) != "system") + .unwrap_or(messages.len()); + let mut messages = messages; + let rest = messages.split_off(leading); + (messages, rest) +} + +/// `translate_system_message` for the leading run. +pub fn system_blocks(run: &[Value]) -> Vec { + let mut blocks = Vec::new(); + for message in run { + match message.get("content") { + Some(Value::String(text)) => { + if text.is_empty() { + continue; + } + let mut block = Map::from_iter([ + ("type".to_owned(), json!("text")), + ("text".to_owned(), json!(text)), + ]); + if let Some(cache_control) = message.get("cache_control") { + block.insert("cache_control".into(), cache_control.clone()); + } + blocks.push(Value::Object(block)); + } + Some(Value::Array(parts)) => { + for part in parts { + let text = part.get("text").cloned().unwrap_or(Value::Null); + let kind = part.get("type").cloned().unwrap_or(Value::Null); + let empty = match &text { + Value::String(text) => text.is_empty(), + Value::Null => true, + _ => false, + }; + if kind.as_str() == Some("text") && empty { + continue; + } + let mut block = + Map::from_iter([("type".to_owned(), kind), ("text".to_owned(), text)]); + if let Some(cache_control) = part.get("cache_control") { + block.insert("cache_control".into(), cache_control.clone()); + } + blocks.push(Value::Object(block)); + } + } + _ => {} + } + } + blocks +} + +/// The non-empty text parts of a system message, each with its normalized `cache_control`. +fn system_text_parts(message: &Value) -> Vec { + let block = |text: &str, cache_control: Option| { + let mut block = Map::from_iter([ + ("type".to_owned(), json!("text")), + ("text".to_owned(), json!(text)), + ]); + if let Some(cache_control) = cache_control { + block.insert("cache_control".into(), cache_control); + } + Value::Object(block) + }; + match message.get("content") { + Some(Value::String(text)) if !text.is_empty() => { + vec![block(text, normalized_cache_control(message))] + } + Some(Value::Array(parts)) => parts + .iter() + .filter(|part| part.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|part| { + let text = part.get("text")?.as_str().filter(|text| !text.is_empty())?; + Some(block(text, normalized_cache_control(part))) + }) + .collect(), + _ => Vec::new(), + } +} + +/// `system_message_as_user` for each message of a run that carries text. +pub fn converted_user_turns(run: &[Value]) -> Vec { + run.iter() + .filter_map(|message| { + let parts = system_text_parts(message); + if parts.is_empty() { + return None; + } + let mut content = vec![json!({"type": "text", "text": CONVERTED_SYSTEM_NOTE})]; + content.extend(parts); + Some(json!({"role": "user", "content": content})) + }) + .collect() +} + +#[derive(Clone, Copy, PartialEq, Eq)] +enum Kind { + System, + Tool, + User, + Other, +} + +fn kind(message: &Value) -> Kind { + match role(message) { + "system" => Kind::System, + "tool" | "function" => Kind::Tool, + "user" => Kind::User, + _ => Kind::Other, + } +} + +/// `_converted_for_unflagged_model`: every later system message becomes a user turn in +/// place, a run caught before a tool result moving behind it. +pub fn place_mid_conversation_system(messages: Vec) -> Vec { + if !messages.iter().any(|message| kind(message) == Kind::System) { + return messages; + } + let mut runs: Vec<(Kind, Vec)> = Vec::new(); + for message in messages { + let message_kind = kind(&message); + match runs.last_mut() { + Some((last, run)) if *last == message_kind => run.push(message), + _ => runs.push((message_kind, vec![message])), + } + } + let mut placed = Vec::new(); + for index in 0..runs.len() { + let (run_kind, run) = &runs[index]; + let follower = runs.get(index + 1).map(|(kind, _)| *kind); + match run_kind { + Kind::System => { + if follower != Some(Kind::Tool) { + placed.extend(converted_user_turns(run)); + } + } + Kind::Tool if index > 0 && runs[index - 1].0 == Kind::System => { + placed.extend(run.iter().cloned()); + placed.extend(converted_user_turns(&runs[index - 1].1)); + } + _ => placed.extend(run.iter().cloned()), + } + } + placed +} + +/// `_sanitize_empty_text_content`. +fn sanitize_empty_text(mut message: Value) -> Value { + if !matches!(role(&message), "user" | "assistant") { + return message; + } + match message.get_mut("content") { + Some(Value::String(text)) if text.trim().is_empty() => { + *text = EMPTY_TEXT_PLACEHOLDER.to_owned(); + } + Some(Value::Array(parts)) => { + for part in parts { + if part.get("type").and_then(Value::as_str) != Some("text") { + continue; + } + let blank = part + .get("text") + .and_then(Value::as_str) + .is_none_or(|text| text.trim().is_empty()); + if blank && let Value::Object(fields) = part { + fields.insert("text".into(), json!(EMPTY_TEXT_PLACEHOLDER)); + } + } + } + _ => {} + } + message +} + +/// `convert_to_anthropic_image_obj`: a data URL as its media type and payload. +fn image_source(url: &str, format: Option<&str>) -> Result<(String, String), Error> { + let (media_type, data) = data_url(url).ok_or_else(|| { + bad_request(format!( + "Image url not in expected format. Example Expected input - \"image_url\": \"data:image/jpeg;base64,{{base64_image}}\". Supported formats - ['image/jpeg', 'image/png', 'image/gif', 'image/webp']. Error: {url:.40}" + )) + })?; + let media_type = match format { + Some(format) => format.to_owned(), + None => media_type.replace("\\/", "/"), + }; + Ok((media_type, data.to_owned())) +} + +/// `create_anthropic_image_param`. +fn image_block(part: &Value) -> Result { + let url = image_url(part).unwrap_or_default(); + let format = part + .get("image_url") + .and_then(|image| image.get("format")) + .and_then(Value::as_str); + if url.starts_with("https://") { + return Ok(json!({"type": "image", "source": {"type": "url", "url": url}})); + } + if url.starts_with("http") { + return Err(Error::Unsupported( + "an http:// image the host did not download", + )); + } + let (media_type, data) = image_source(url, format)?; + Ok( + json!({"type": "image", "source": {"type": "base64", "media_type": media_type, "data": data}}), + ) +} + +/// `anthropic_process_openai_file_message`. +fn file_block(part: &Value) -> Result { + let file = part.get("file").ok_or_else(|| { + bad_request("Content block has type='file' but is missing the required 'file' field") + })?; + let format = file.get("format").and_then(Value::as_str); + if let Some(data) = file + .get("file_data") + .and_then(Value::as_str) + .filter(|data| !data.is_empty()) + { + if data.starts_with("http") { + return Err(Error::Unsupported( + "a file_data URL the host did not download", + )); + } + let (media_type, data) = image_source(data, format)?; + return Ok( + json!({"type": "document", "source": {"type": "base64", "media_type": media_type, "data": data}}), + ); + } + match file + .get("file_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + { + Some(id) if format.is_none() && id.starts_with("http") => { + Ok(json!({"type": "document", "source": {"type": "url", "url": id}})) + } + // Uploaded files need the Files API betas and, for a container upload, the code + // execution tool. + Some(_) => Err(Error::Unsupported("an Anthropic Files API file_id")), + None => Err(bad_request(format!( + "Either file_data or file_id must be present in the file message: {part}" + ))), + } +} + +/// `_is_anthropic_document_data_uri`. +fn is_document_data_uri(url: &str) -> bool { + url.strip_prefix("data:") + .map(|rest| rest.split([';', ',']).next().unwrap_or_default()) + .is_some_and(|media_type| matches!(media_type, "application/pdf" | "text/plain")) +} + +/// `convert_to_anthropic_tool_result`. +fn tool_result(message: &Value) -> Result { + let content = match message.get("content") { + Some(Value::String(text)) => json!(text), + Some(Value::Array(parts)) => { + let mut blocks = Vec::new(); + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("text") => { + let mut block = Map::from_iter([ + ("type".to_owned(), json!("text")), + ( + "text".to_owned(), + part.get("text").cloned().unwrap_or(Value::Null), + ), + ]); + if let Some(cache_control) = + part.get("cache_control").filter(|value| !value.is_null()) + { + block.insert("cache_control".into(), cache_control.clone()); + } + blocks.push(Value::Object(block)); + } + Some("image_url") => { + let url = image_url(part).unwrap_or_default(); + let block = match is_document_data_uri(url) { + true => { + file_block(&json!({"type": "file", "file": {"file_data": url}}))? + } + false => image_block(part)?, + }; + blocks.push(with_cache_control(block, part)); + } + Some("tool_reference") => blocks.push(json!({ + "type": "tool_reference", + "tool_name": part.get("tool_name").cloned().unwrap_or(Value::Null), + })), + Some("file") => blocks.push(with_cache_control(file_block(part)?, part)), + _ => {} + } + } + Value::Array(blocks) + } + _ => json!(""), + }; + let id = match role(message) { + "tool" => message + .get("tool_call_id") + .and_then(Value::as_str) + .map(str::to_owned) + .ok_or_else(|| bad_request("A tool message needs a tool_call_id."))?, + _ => message + .get("tool_call_id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + .map(str::to_owned) + .unwrap_or_else(|| uuid::Uuid::new_v4().to_string()), + }; + let mut result = Map::from_iter([ + ("type".to_owned(), json!("tool_result")), + ("tool_use_id".to_owned(), json!(sanitize_tool_use_id(&id))), + ("content".to_owned(), content), + ]); + if let Some(cache_control) = message + .get("cache_control") + .filter(|value| !value.is_null()) + { + result.insert("cache_control".into(), cache_control.clone()); + } + Ok(Value::Object(result)) +} + +pub fn is_encrypted_reasoning(block: &Value) -> bool { + let field = match block.get("type").and_then(Value::as_str) { + Some("thinking") => block.get("signature"), + Some("redacted_thinking") => block.get("data"), + _ => None, + }; + field + .and_then(Value::as_str) + .is_some_and(|value| value.starts_with(ENCRYPTED_REASONING_SIGNATURE_PREFIX)) +} + +/// `is_unsignable_thinking_block`. +pub fn is_unsignable_thinking_block(block: &Value) -> bool { + if is_encrypted_reasoning(block) { + return true; + } + block.get("type").and_then(Value::as_str) == Some("thinking") + && block + .get("signature") + .and_then(Value::as_str) + .is_none_or(str::is_empty) +} + +/// `native_redacted_thinking_data`: the base64 wrapper Converse adds, removed. +pub fn native_redacted_thinking_data(data: &str) -> String { + let engine = base64::engine::general_purpose::STANDARD; + engine + .decode(data) + .ok() + .and_then(|bytes| String::from_utf8(bytes).ok()) + .filter(|payload| payload.is_ascii() && engine.decode(payload).is_ok()) + .unwrap_or_else(|| data.to_owned()) +} + +/// `_replayable_thinking_blocks`. +fn replayable_thinking_blocks(blocks: &[Value]) -> Vec { + blocks + .iter() + .filter(|block| !is_unsignable_thinking_block(block)) + .map(|block| { + let mut block = block.clone(); + if let Value::Object(fields) = &mut block { + if fields.get("type").and_then(Value::as_str) == Some("redacted_thinking") + && let Some(Value::String(data)) = fields.get("data") + { + let native = native_redacted_thinking_data(data); + fields.insert("data".into(), json!(native)); + } + fields.remove("index"); + } + block + }) + .collect() +} + +/// The tool calls of an assistant message, as `tool_use` blocks. +fn tool_uses(tool_calls: &[Value], seen: &mut HashSet) -> Result, Error> { + let mut blocks = Vec::new(); + for call in tool_calls { + if call.get("type").and_then(Value::as_str) != Some("function") { + continue; + } + let id = call.get("id").and_then(Value::as_str).unwrap_or_default(); + if id.starts_with("srvtoolu_") { + return Err(Error::Unsupported( + "replaying an Anthropic server tool call", + )); + } + let function = call.get("function"); + let name = function + .and_then(|function| function.get("name")) + .cloned() + .unwrap_or(Value::Null); + let input = parse_tool_arguments(function.and_then(|function| function.get("arguments")))?; + let id = sanitize_tool_use_id(id); + let block = with_cache_control( + json!({"type": "tool_use", "id": id, "name": name, "input": input}), + call, + ); + if !seen.insert(id) { + continue; + } + blocks.push(block); + } + Ok(blocks) +} + +/// `anthropic_messages_pt` over messages whose later system messages are already placed. +pub fn anthropic_messages_pt(messages: Vec) -> Result, Error> { + let messages: Vec = messages.into_iter().map(sanitize_empty_text).collect(); + if messages.is_empty() { + return Err(bad_request( + "Anthropic requires at least one non-system message. Either provide one, or set `litellm.modify_params = True` // `litellm_settings::modify_params: True` to add the dummy user message - {'role': 'user', 'content': 'Please continue.'}.", + )); + } + let user_types = |message: &Value| matches!(role(message), "user" | "tool" | "function"); + // `_tool_result_later_in_turn`: whether a client tool result follows in the same turn. + let mut tool_result_later = vec![false; messages.len()]; + let mut later = false; + for index in (0..messages.len()).rev() { + tool_result_later[index] = later; + let message = &messages[index]; + later = user_types(message) && (later || matches!(role(message), "tool" | "function")); + } + + let mut out: Vec = Vec::new(); + let mut index = 0; + while index < messages.len() { + let start = index; + let mut user_content: Vec = Vec::new(); + while index < messages.len() && user_types(&messages[index]) { + let message = &messages[index]; + if role(message) == "user" { + match message.get("content") { + _ if content_is_empty(message) => { + if !tool_result_later[index] + && message.get("cache_control").is_some_and(truthy) + { + user_content.push(with_cache_control( + json!({"type": "text", "text": EMPTY_TEXT_PLACEHOLDER}), + message, + )); + } + } + Some(Value::Array(parts)) => { + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("image_url") => { + user_content.push(with_cache_control(image_block(part)?, part)) + } + Some("text") => user_content.push(with_cache_control( + json!({"type": "text", "text": part.get("text").cloned().unwrap_or(Value::Null)}), + part, + )), + Some("document") => { + user_content.push(with_cache_control(part.clone(), part)) + } + Some("file") => { + user_content.push(with_cache_control(file_block(part)?, part)) + } + _ => {} + } + } + } + Some(Value::String(text)) => user_content.push(with_cache_control( + json!({"type": "text", "text": text}), + message, + )), + _ => {} + } + } else { + user_content.push(tool_result(message)?); + } + index += 1; + } + if !user_content.is_empty() { + out.push(json!({"role": "user", "content": user_content})); + } else if let Some(empty) = messages[start..index] + .iter() + .rev() + .find(|message| role(message) == "user" && content_is_empty(message)) + { + out.push(json!({"role": "user", "content": [with_cache_control( + json!({"type": "text", "text": EMPTY_TEXT_PLACEHOLDER}), + empty, + )]})); + } + + let mut seen_tool_ids: HashSet = HashSet::new(); + let mut assistant_content: Vec = Vec::new(); + while index < messages.len() && role(&messages[index]) == "assistant" { + let message = &messages[index]; + let provider_fields = message.get("provider_specific_fields"); + if let Some(Value::Array(blocks)) = + provider_fields.and_then(|fields| fields.get("compaction_blocks")) + { + assistant_content.extend(blocks.iter().cloned()); + } + if provider_fields.is_some_and(|fields| { + fields + .get("web_search_results") + .is_some_and(|v| !v.is_null()) + || fields.get("tool_results").is_some_and(|v| !v.is_null()) + }) && message + .get("tool_calls") + .and_then(Value::as_array) + .is_some_and(|calls| !calls.is_empty()) + { + return Err(Error::Unsupported( + "replaying Anthropic server tool results", + )); + } + let thinking_blocks = match message.get("thinking_blocks") { + Some(Value::Array(blocks)) => Some(replayable_thinking_blocks(blocks)), + _ => None, + }; + let parts = message.get("content").and_then(Value::as_array); + let inline_thinking = parts.is_some_and(|parts| { + parts.iter().any(|part| { + matches!( + part.get("type").and_then(Value::as_str), + Some("thinking" | "redacted_thinking") + ) + }) + }); + if let Some(blocks) = thinking_blocks + && !inline_thinking + { + assistant_content.extend(blocks); + } + match message.get("content") { + Some(Value::Array(parts)) => { + for part in parts.iter().filter(|part| part.is_object()) { + let kind = part.get("type").and_then(Value::as_str).unwrap_or_default(); + let text = part.get("text").and_then(Value::as_str).unwrap_or_default(); + if kind == "thinking" && !is_unsignable_thinking_block(part) { + assistant_content.push(part.clone()); + } else if kind == "text" && !text.is_empty() { + assistant_content.push(with_cache_control( + json!({"type": "text", "text": text}), + part, + )); + } else if kind == "server_tool_use" || kind.ends_with("_tool_result") { + assistant_content.push(part.clone()); + } + } + } + Some(Value::String(text)) if !text.is_empty() => { + assistant_content.push(with_cache_control( + json!({"type": "text", "text": text}), + message, + )); + } + _ => {} + } + if let Some(Value::Array(calls)) = message.get("tool_calls") { + assistant_content.extend(tool_uses(calls, &mut seen_tool_ids)?); + } + if let Some(call) = message.get("function_call").filter(|call| !call.is_null()) { + let input = parse_tool_arguments(call.get("arguments"))?; + assistant_content.push(json!({ + "type": "tool_use", + "id": uuid::Uuid::new_v4().to_string(), + "name": call.get("name").filter(|name| !name.is_null()).cloned().unwrap_or_else(|| json!("")), + "input": input, + })); + } + index += 1; + } + if !assistant_content.is_empty() { + out.push(json!({"role": "assistant", "content": assistant_content})); + } + if index == start { + return Err(bad_request(format!( + "Invalid Message passed in {}", + messages[index] + ))); + } + } + if let Some(last) = out.last_mut() + && last["role"] == "assistant" + && let Some(Value::Array(blocks)) = last.get_mut("content") + { + for block in blocks { + if block.get("type").and_then(Value::as_str) == Some("text") + && let Some(Value::String(text)) = block.get_mut("text") + { + *text = text.trim_end().to_owned(); + } + } + } + Ok(out) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn list(value: Value) -> Vec { + value.as_array().unwrap().clone() + } + + #[rstest] + fn tool_round_trip_merges_turns() { + let messages = anthropic_messages_pt(list(json!([ + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": null, "tool_calls": [{"id": "call.1", "type": "function", "function": {"name": "get", "arguments": "{\"c\": \"P\"}"}}]}, + {"role": "tool", "tool_call_id": "call.1", "content": "sunny"}, + {"role": "user", "content": [{"type": "text", "text": "thanks", "cache_control": {"type": "ephemeral"}}]} + ]))) + .unwrap(); + assert_eq!( + messages, + list(json!([ + {"role": "user", "content": [{"type": "text", "text": "weather?"}]}, + {"role": "assistant", "content": [{"type": "tool_use", "id": "call_1", "name": "get", "input": {"c": "P"}}]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"}, + {"type": "text", "text": "thanks", "cache_control": {"type": "ephemeral"}} + ]} + ])) + ); + } + + #[rstest] + fn images_empty_text_and_final_assistant_whitespace() { + let messages = anthropic_messages_pt(list(json!([ + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBO"}}, + {"type": "image_url", "image_url": "https://x/y.png"}, + {"type": "text", "text": " "} + ]}, + {"role": "assistant", "content": "Sure "} + ]))) + .unwrap(); + assert_eq!( + messages[0]["content"], + json!([ + {"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBO"}}, + {"type": "image", "source": {"type": "url", "url": "https://x/y.png"}}, + {"type": "text", "text": EMPTY_TEXT_PLACEHOLDER} + ]) + ); + assert_eq!( + messages[1]["content"], + json!([{"type": "text", "text": "Sure"}]) + ); + } + + #[rstest] + fn later_system_messages_become_operator_notes() { + let placed = place_mid_conversation_system(list(json!([ + {"role": "user", "content": "a"}, + {"role": "assistant", "content": "b", "tool_calls": [{"id": "t", "type": "function", "function": {"name": "f", "arguments": "{}"}}]}, + {"role": "system", "content": "remember", "cache_control": {"type": "ephemeral", "ttl": "1h", "x": 1}}, + {"role": "tool", "tool_call_id": "t", "content": "r"} + ]))); + assert_eq!(placed[2]["role"], "tool"); + assert_eq!( + placed[3], + json!({"role": "user", "content": [ + {"type": "text", "text": CONVERTED_SYSTEM_NOTE}, + {"type": "text", "text": "remember", "cache_control": {"type": "ephemeral", "ttl": "1h"}} + ]}) + ); + } + + #[rstest] + fn unsigned_thinking_is_dropped_and_signed_thinking_leads() { + let messages = anthropic_messages_pt(list(json!([ + {"role": "user", "content": "q"}, + {"role": "assistant", "content": "a", "thinking_blocks": [ + {"type": "thinking", "thinking": "t", "signature": "s", "index": 0}, + {"type": "thinking", "thinking": "u"} + ]} + ]))) + .unwrap(); + assert_eq!( + messages[1]["content"], + json!([{"type": "thinking", "thinking": "t", "signature": "s"}, {"type": "text", "text": "a"}]) + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/chat/mod.rs b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs index f0050b7dc71..955055f2fbd 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/mod.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/mod.rs @@ -1,2 +1,4 @@ pub mod handler; +pub mod messages; +pub mod params; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/anthropic/chat/params.rs b/litellm-rust/crates/llms/src/anthropic/chat/params.rs new file mode 100644 index 00000000000..be507b1c46c --- /dev/null +++ b/litellm-rust/crates/llms/src/anthropic/chat/params.rs @@ -0,0 +1,1293 @@ +//! Python's `AnthropicConfig.map_openai_params` and the model-map reads behind it: the +//! OpenAI params a chat request carries, as the Messages API takes them. + +use serde_json::{Map, Value, json}; + +use crate::{ + Error, ErrorDetail, + base_llm::chat::{ + params::{ + ChatRequestContext, DEFAULT_MAX_TOKENS, DropPolicy, RESPONSE_FORMAT_TOOL_NAME, + capability, ordered, truthy, unsupported_value, + }, + response_format::{ResponseFormatSchema, unpack_defs, unpack_legacy_defs}, + }, + bedrock::chat::converse_params::bedrock_base_model, +}; + +/// `ANTHROPIC_MIN_THINKING_BUDGET_TOKENS`. +pub const MIN_THINKING_BUDGET_TOKENS: i64 = 1024; + +/// The thinking budgets `_map_reasoning_effort` gives a legacy-thinking model. +pub fn reasoning_effort_budget(effort: &str) -> Option { + Some(match effort { + "low" => 1024, + "medium" => 2048, + "high" => 4096, + "xhigh" => 8192, + "max" => 16384, + // `max(DEFAULT_REASONING_EFFORT_MINIMAL_THINKING_BUDGET, ANTHROPIC_MIN_THINKING_BUDGET_TOKENS)` + "minimal" => 128.max(MIN_THINKING_BUDGET_TOKENS), + _ => return None, + }) +} + +/// `REASONING_EFFORT_TO_OUTPUT_CONFIG_EFFORT`. +pub fn output_config_effort(effort: &str) -> Option<&'static str> { + Some(match effort { + "low" | "minimal" => "low", + "medium" => "medium", + "high" => "high", + "xhigh" => "xhigh", + "max" => "max", + _ => return None, + }) +} + +pub fn unmapped_reasoning_effort(effort: &str) -> Error { + Error::InvalidRequest(ErrorDetail::InvalidChoice { + field: "reasoning effort", + actual: format!("'{effort}'"), + choices: vec!["minimal", "low", "medium", "high", "xhigh", "max", "none"], + }) +} + +/// Strips a trailing `-v` or `-v:`. +fn strip_bedrock_version(model: &str) -> String { + let bytes = model.as_bytes(); + let mut end = bytes.len(); + // Optional `:`. + let mut cut = end; + while cut > 0 && bytes[cut - 1].is_ascii_digit() { + cut -= 1; + } + if cut < end && cut > 0 && bytes[cut - 1] == b':' { + end = cut - 1; + } + let mut digits = end; + while digits > 0 && bytes[digits - 1].is_ascii_digit() { + digits -= 1; + } + if digits < end && digits >= 2 && &bytes[digits - 2..digits] == b"-v" { + return model[..digits - 2].to_owned(); + } + // The `:` alone is not a version suffix, so only the full shape is removed. + model.to_owned() +} + +/// Strips a trailing `:`. +fn strip_profile_minor(model: &str) -> String { + let trimmed = model.trim_end_matches(|c: char| c.is_ascii_digit()); + match trimmed.strip_suffix(':') { + Some(head) if trimmed.len() < model.len() => head.to_owned(), + _ => model.to_owned(), + } +} + +/// Strips a trailing `-<8 digits>`. +fn strip_dated_release(model: &str) -> String { + let bytes = model.as_bytes(); + if bytes.len() >= 9 { + let tail = &bytes[bytes.len() - 9..]; + if tail[0] == b'-' && tail[1..].iter().all(u8::is_ascii_digit) { + return model[..model.len() - 9].to_owned(); + } + } + model.to_owned() +} + +/// `(\d)\.(\d)` -> `\1-\2`, non-overlapping like `re.sub`. +fn hyphenate_dotted_versions(model: &str) -> String { + let chars: Vec = model.chars().collect(); + let mut out = String::with_capacity(model.len()); + let mut index = 0; + while index < chars.len() { + if index + 2 < chars.len() + && chars[index].is_ascii_digit() + && chars[index + 1] == '.' + && chars[index + 2].is_ascii_digit() + { + out.push(chars[index]); + out.push('-'); + out.push(chars[index + 2]); + index += 3; + } else { + out.push(chars[index]); + index += 1; + } + } + out +} + +fn strip_at_version(model: &str) -> String { + match model.rfind('@') { + Some(at) if at > 0 => model[..at].to_owned(), + _ => model.to_owned(), + } +} + +/// `AnthropicModelInfo._model_map_lookup_candidates`. +pub fn model_map_candidates(model: &str) -> Vec { + let mut primary = vec![model.to_owned()]; + for prefix in [ + "bedrock/converse/", + "bedrock/invoke/", + "bedrock/", + "vertex_ai/", + ] { + if let Some(rest) = model.strip_prefix(prefix) { + primary.push(rest.to_owned()); + } + } + let base = bedrock_base_model(model); + if !base.is_empty() { + primary.push(base.clone()); + primary.push(format!("bedrock/{base}")); + } + let mut candidates: Vec = Vec::new(); + let mut push = |candidate: String| { + if !candidates.contains(&candidate) { + candidates.push(candidate); + } + }; + for candidate in &primary { + push(candidate.clone()); + } + for candidate in &primary { + push(strip_bedrock_version(candidate)); + push(strip_profile_minor(candidate)); + push(strip_dated_release(candidate)); + push(hyphenate_dotted_versions(candidate)); + push(strip_dated_release(&strip_bedrock_version( + &hyphenate_dotted_versions(candidate), + ))); + push(strip_at_version(candidate)); + } + candidates +} + +/// The cost-map reads a model's request rules make, through its lookup candidates. +pub struct ModelCapabilities<'a> { + pub model: String, + pub context: ChatRequestContext<'a>, + pub candidates: Vec, +} + +impl<'a> ModelCapabilities<'a> { + pub fn new(model: &str, context: &ChatRequestContext<'a>) -> Self { + Self { + model: model.to_owned(), + context: *context, + candidates: model_map_candidates(model), + } + } + + /// `_get_model_capability`: the flag the first candidate the map knows sets. + pub fn flag(&self, key: &str) -> Result, Error> { + capability(&self.context, &self.candidates, key) + } + + /// `_supports_model_capability`: the flag is set to `true`. + pub fn supports(&self, key: &str) -> Result { + Ok(self.flag(key)? == Some(true)) + } + + /// `_supports_sampling_params`. + pub fn supports_sampling(&self) -> Result { + if let Some(flag) = self.flag("supports_sampling_params")? { + return Ok(flag); + } + let lower = self.model.to_lowercase(); + Ok(![ + "fable", "opus-4-7", "opus_4_7", "opus-4.7", "opus_4.7", "opus-4-8", "opus_4_8", + "opus-4.8", "opus_4.8", + ] + .iter() + .any(|name| lower.contains(name))) + } + + pub fn forced_tool_use_unsupported(&self) -> Result { + Ok(self.flag("supports_forced_tool_use")? == Some(false)) + } + + pub fn is_adaptive(&self) -> Result { + self.supports("supports_adaptive_thinking") + } +} + +/// Python's `value` in an error message: its `str()`. +pub fn python_str(value: &Value) -> String { + match value { + Value::String(text) => text.clone(), + Value::Bool(true) => "True".into(), + Value::Bool(false) => "False".into(), + Value::Null => "None".into(), + other => other.to_string(), + } +} + +/// `_apply_sampling_param`: the param, unless the model removed sampling params. +pub fn apply_sampling_param( + out: &mut Map, + caps: &ModelCapabilities<'_>, + drop: DropPolicy, + param: &str, + value: &Value, + output_key: &str, +) -> Result<(), Error> { + if caps.supports_sampling()? || (param == "temperature" && value.as_f64() == Some(1.0)) { + out.insert(output_key.to_owned(), value.clone()); + return Ok(()); + } + drop.unsupported(|| { + Error::InvalidRequest(ErrorDetail::UnsupportedParameter { + model: caps.model.clone(), + param: param.to_owned(), + value: python_str(value), + hint: match param { + "temperature" => "Only temperature=1 is supported. ".into(), + _ => String::new(), + }, + }) + }) +} + +/// `_valid_user_id`: Anthropic rejects an email or a phone number as `metadata.user_id`. +pub fn valid_user_id(user_id: &str) -> bool { + !(looks_like_email(user_id) || looks_like_phone(user_id)) +} + +fn looks_like_email(text: &str) -> bool { + let Some((local, domain)) = text.split_once('@') else { + return false; + }; + let local_ok = !local.is_empty() + && local + .chars() + .all(|c| c.is_ascii_alphanumeric() || "._%+-".contains(c)); + let domain_ok = domain + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '.' || c == '-') + && domain.rsplit_once('.').is_some_and(|(head, tld)| { + !head.is_empty() && tld.len() >= 2 && tld.chars().all(|c| c.is_ascii_alphabetic()) + }); + local_ok && domain_ok +} + +fn looks_like_phone(text: &str) -> bool { + let body = text.strip_prefix('+').unwrap_or(text); + body.chars().count() >= 7 + && body + .chars() + .all(|c| c.is_numeric() || c.is_whitespace() || "()-".contains(c)) +} + +/// `_map_stop_sequences`: whitespace-only entries are dropped only under `drop_params`. +pub fn map_stop_sequences(stop: &Value, drop: DropPolicy) -> Option { + let keep = |text: &str| !(is_python_space(text) && drop.drops()); + match stop { + Value::String(text) => keep(text).then(|| json!([text])), + Value::Array(items) => { + let kept: Vec = items + .iter() + .filter(|item| item.as_str().is_none_or(keep)) + .cloned() + .collect(); + (!kept.is_empty()).then_some(Value::Array(kept)) + } + _ => None, + } +} + +/// `str.isspace()`: non-empty and all whitespace. +pub fn is_python_space(text: &str) -> bool { + !text.is_empty() && text.chars().all(char::is_whitespace) +} + +/// `sanitize_input_schema_for_anthropic`. +pub fn sanitize_input_schema(schema: Option<&Value>) -> Value { + let mut normalized = match schema { + Some(Value::Object(object)) => object.clone(), + _ => Map::new(), + }; + if normalized.get("type").and_then(Value::as_str) != Some("object") { + normalized.insert("type".into(), json!("object")); + } + normalized + .entry("properties") + .or_insert_with(|| Value::Object(Map::new())); + unpack_legacy_defs(&mut normalized); + Value::Object( + normalized + .into_iter() + .filter(|(key, _)| { + matches!( + key.as_str(), + "type" + | "properties" + | "additionalProperties" + | "required" + | "$defs" + | "strict" + ) + }) + .collect(), + ) +} + +fn invalid_tool(message: &str) -> Error { + unsupported_value(message.to_owned()) +} + +/// `_map_tool_helper` for the tool shapes chat translates: function and custom tools. +fn map_function_tool(tool: &Map) -> Result { + let function = tool.get("function").and_then(Value::as_object); + let Some(function) = function else { + return Err(invalid_tool("A function tool needs a `function` object.")); + }; + let name = function + .get("name") + .cloned() + .ok_or_else(|| invalid_tool("A function tool needs a name."))?; + let default_schema = json!({"type": "object", "properties": {}}); + let schema = sanitize_input_schema(Some(function.get("parameters").unwrap_or(&default_schema))); + let mut mapped = Map::new(); + mapped.insert("name".into(), name); + mapped.insert("input_schema".into(), schema); + mapped.insert("type".into(), json!("custom")); + let eager = match tool.get("eager_input_streaming") { + Some(Value::Bool(flag)) => Some(*flag), + Some(Value::Null) | None => match function.get("eager_input_streaming") { + Some(Value::Bool(flag)) => Some(*flag), + Some(Value::Null) | None => None, + Some(_) => return Err(invalid_tool("eager_input_streaming must be a boolean")), + }, + Some(_) => return Err(invalid_tool("eager_input_streaming must be a boolean")), + }; + if let Some(eager) = eager { + mapped.insert("eager_input_streaming".into(), json!(eager)); + } + if let Some(description) = function.get("description").filter(|d| !d.is_null()) { + mapped.insert("description".into(), description.clone()); + } + match (tool.get("cache_control"), function.get("cache_control")) { + (Some(cache_control), _) if !cache_control.is_null() => { + mapped.insert("cache_control".into(), cache_control.clone()); + } + (_, Some(Value::Object(cache_control))) => { + mapped.insert("cache_control".into(), Value::Object(cache_control.clone())); + } + _ => {} + } + match (tool.get("defer_loading"), function.get("defer_loading")) { + (Some(Value::Bool(flag)), _) => { + mapped.insert("defer_loading".into(), json!(flag)); + } + (Some(other), _) if !other.is_null() => { + return Err(invalid_tool("defer_loading must be a boolean")); + } + (_, Some(Value::Bool(flag))) => { + mapped.insert("defer_loading".into(), json!(flag)); + } + (_, Some(other)) if !other.is_null() => { + return Err(invalid_tool("defer_loading must be a boolean")); + } + _ => {} + } + let string_list = |value: &Value| { + value + .as_array() + .is_some_and(|items| items.iter().all(Value::is_string)) + }; + for key in ["allowed_callers", "input_examples"] { + let value = tool + .get(key) + .filter(|v| !v.is_null()) + .or_else(|| function.get(key).filter(|v| !v.is_null())); + if let Some(value) = value { + if key == "allowed_callers" && !string_list(value) { + return Err(invalid_tool("allowed_callers must be a list of strings")); + } + if key == "input_examples" && !value.is_array() { + return Err(invalid_tool("input_examples must be a list")); + } + mapped.insert(key.into(), value.clone()); + } + } + Ok(Value::Object(mapped)) +} + +/// `_map_tools` for function and custom tools and tools already in Anthropic's shape. +pub fn map_tools(tools: &Value) -> Result, Error> { + let Some(tools) = tools.as_array() else { + return Err(invalid_tool("tools must be a list")); + }; + let mut mapped = Vec::new(); + for tool in tools { + let Some(object) = tool.as_object() else { + return Err(invalid_tool("each tool must be an object")); + }; + if object.contains_key("input_schema") { + mapped.push(tool.clone()); + continue; + } + match object.get("type").and_then(Value::as_str) { + Some("function" | "custom") => mapped.push(map_function_tool(object)?), + Some("namespace") => { + for nested in object + .get("tools") + .and_then(Value::as_array) + .into_iter() + .flatten() + { + let Some(nested_object) = nested.as_object() else { + continue; + }; + if nested_object.contains_key("input_schema") { + mapped.push(nested.clone()); + } else if !nested_object.contains_key("function") + && nested_object.contains_key("name") + { + let kind = nested_object + .get("type") + .cloned() + .unwrap_or_else(|| json!("function")); + let function: Map = nested_object + .iter() + .filter(|(key, _)| key.as_str() != "type") + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + let wrapped = Map::from_iter([ + ("type".to_owned(), kind.clone()), + ("function".to_owned(), Value::Object(function)), + ]); + match kind.as_str() { + Some("function" | "custom") => { + mapped.push(map_function_tool(&wrapped)?) + } + _ => { + return Err(Error::Unsupported("a hosted tool inside a namespace")); + } + } + } else if nested_object.contains_key("function") { + match nested_object.get("type").and_then(Value::as_str) { + Some("function" | "custom") => { + mapped.push(map_function_tool(nested_object)?) + } + _ => { + return Err(Error::Unsupported("a hosted tool inside a namespace")); + } + } + } + } + } + // Computer use, hosted tools, MCP servers and tool search each have their own + // shape and betas; they stay on Python's path. + _ => { + return Err(Error::Unsupported( + "a chat tool other than a function or custom tool", + )); + } + } + } + Ok(mapped) +} + +/// `_map_tool_choice`. +pub fn map_tool_choice(tool_choice: Option<&Value>, parallel: Option<&Value>) -> Option { + let mut choice: Option> = match tool_choice { + Some(Value::String(choice)) => match choice.as_str() { + "auto" => Some(json!({"type": "auto"})), + "required" => Some(json!({"type": "any"})), + "none" => Some(json!({"type": "none"})), + _ => None, + }, + Some(Value::Object(choice)) + if choice.contains_key("type") && !choice.contains_key("function") => + { + match choice.get("type").and_then(Value::as_str) { + Some("auto") => Some(json!({"type": "auto"})), + Some("required" | "any") => Some(json!({"type": "any"})), + Some("none") => Some(json!({"type": "none"})), + _ => None, + } + } + Some(Value::Object(choice)) => choice + .get("function") + .and_then(|function| function.get("name")) + .filter(|name| !name.is_null()) + .map(|name| json!({"type": "tool", "name": name})), + _ => None, + } + .and_then(|value| value.as_object().cloned()); + if let Some(parallel) = parallel.filter(|parallel| !parallel.is_null()) { + let disable = json!(!truthy(parallel)); + let choice_is_none = tool_choice.and_then(Value::as_str) == Some("none"); + match &mut choice { + _ if choice_is_none => {} + Some(choice) => { + choice.insert("disable_parallel_tool_use".into(), disable); + } + None => { + choice = Some(Map::from_iter([ + ("type".to_owned(), json!("auto")), + ("disable_parallel_tool_use".to_owned(), disable), + ])); + } + } + } + choice.map(Value::Object) +} + +/// `_apply_forced_tool_choice`: `any` / `tool` on a model that refuses forced tool use. +pub fn apply_forced_tool_choice( + choice: Value, + caps: &ModelCapabilities<'_>, + drop: DropPolicy, +) -> Result { + let kind = choice.get("type").and_then(Value::as_str); + if !matches!(kind, Some("any" | "tool")) || !caps.forced_tool_use_unsupported()? { + return Ok(choice); + } + drop.unsupported(|| { + unsupported_value(format!( + "{} does not support forced tool use (tool_choice='required' or a named tool). Use tool_choice='auto' and tell the model in the prompt when to call the tool, or set `litellm.drop_params = True` to downgrade to 'auto' automatically.", + caps.model + )) + })?; + let mut auto = Map::from_iter([("type".to_owned(), json!("auto"))]); + if let Some(disable) = choice.get("disable_parallel_tool_use") { + auto.insert("disable_parallel_tool_use".into(), disable.clone()); + } + Ok(Value::Object(auto)) +} + +fn enum_conflicts_with_declared_type(schema: &Map) -> bool { + let (Some(Value::Array(values)), Some(declared)) = (schema.get("enum"), schema.get("type")) + else { + return false; + }; + if declared.is_array() { + return true; + } + let check: fn(&Value) -> bool = match declared.as_str() { + Some("null") => Value::is_null, + Some("boolean") => Value::is_boolean, + Some("integer") => |v| v.is_i64() || v.is_u64(), + Some("number") => Value::is_number, + Some("string") => Value::is_string, + Some("array") => Value::is_array, + Some("object") => Value::is_object, + _ => return false, + }; + !values.iter().all(check) +} + +const CONSTRAINT_LABELS: [(&str, &str); 25] = [ + ("minItems", "minimum number of items: {}"), + ("maxItems", "maximum number of items: {}"), + ("uniqueItems", "all array items must be unique"), + ("contains", "array must contain an item matching: {}"), + ("minContains", "minimum number of matching items: {}"), + ("maxContains", "maximum number of matching items: {}"), + ("prefixItems", "leading items must match, in order: {}"), + ("minimum", "minimum value: {}"), + ("maximum", "maximum value: {}"), + ("exclusiveMinimum", "exclusive minimum value: {}"), + ("exclusiveMaximum", "exclusive maximum value: {}"), + ("multipleOf", "must be a multiple of {}"), + ("minLength", "minimum length: {}"), + ("maxLength", "maximum length: {}"), + ("minProperties", "minimum number of properties: {}"), + ("maxProperties", "maximum number of properties: {}"), + ( + "patternProperties", + "properties whose names match each pattern must satisfy: {}", + ), + ("propertyNames", "property names must satisfy: {}"), + ("dependentRequired", "dependent required properties: {}"), + ("dependentSchemas", "dependent schemas: {}"), + ( + "unevaluatedProperties", + "unevaluated properties must satisfy: {}", + ), + ("if", "conditional (if): {}"), + ("then", "conditional (then): {}"), + ("else", "conditional (else): {}"), + ("not", "must not match: {}"), +]; + +/// `filter_anthropic_output_schema`: the constraints structured output cannot enforce, +/// removed and described instead. +pub fn filter_output_schema(schema: &Value) -> Value { + let Some(object) = schema.as_object() else { + return schema.clone(); + }; + let notes: Vec = CONSTRAINT_LABELS + .iter() + .filter_map(|(field, label)| { + let value = object.get(*field)?; + if value == &Value::Bool(false) { + return None; + } + let rendered = match value { + Value::Object(_) | Value::Array(_) => { + crate::base_llm::chat::params::python_dumps(value) + } + other => python_str(other), + }; + Some(label.replace("{}", &rendered)) + }) + .collect(); + let mut result = Map::new(); + if !notes.is_empty() { + let note = format!("Note: {}.", notes.join(", ")); + let existing = object + .get("description") + .and_then(Value::as_str) + .unwrap_or_default(); + let description = match existing.is_empty() { + true => note, + false => format!("{existing} {note}"), + }; + result.insert("description".into(), json!(description)); + } + let drops_type = enum_conflicts_with_declared_type(object); + for (key, value) in object { + if CONSTRAINT_LABELS.iter().any(|(field, _)| field == key) + || (key == "type" && drops_type) + || (key == "description" && result.contains_key("description")) + { + continue; + } + match (key.as_str(), value) { + ("properties" | "$defs", Value::Object(nested)) => { + result.insert( + key.clone(), + Value::Object( + nested + .iter() + .map(|(k, v)| (k.clone(), filter_output_schema(v))) + .collect(), + ), + ); + } + ("items", Value::Object(_)) => { + result.insert(key.clone(), filter_output_schema(value)); + } + ("anyOf" | "oneOf", Value::Array(items)) => { + let mut any_of = match result.remove("anyOf") { + Some(Value::Array(existing)) => existing, + _ => Vec::new(), + }; + any_of.extend(items.iter().map(filter_output_schema)); + result.insert("anyOf".into(), Value::Array(any_of)); + } + ("allOf", Value::Array(items)) => { + result.insert( + key.clone(), + Value::Array(items.iter().map(filter_output_schema).collect()), + ); + } + _ => { + result.insert(key.clone(), value.clone()); + } + } + } + if result.get("type").and_then(Value::as_str) == Some("object") { + result.insert("additionalProperties".into(), json!(false)); + } + Value::Object(result) +} + +/// The schema a `response_format` names, if any. A `json_schema` without `schema` names none. +fn response_format_schema(value: &Map) -> Option { + if let Some(schema) = value.get("response_schema") { + return Some(schema.clone()).filter(|schema| !schema.is_null()); + } + value + .get("json_schema") + .and_then(|json_schema| json_schema.get("schema")) + .filter(|schema| !schema.is_null()) + .cloned() +} + +/// `map_response_format_to_anthropic_output_format`. +fn output_format(value: &Map) -> Option { + let mut schema = response_format_schema(value)?; + let mut defs = Map::new(); + if let Value::Object(object) = &mut schema { + let definitions = object.remove("definitions"); + let dollar_defs = object.remove("$defs"); + if let Some(Value::Object(found)) = dollar_defs.or(definitions) { + defs = found; + } + } + if !defs.is_empty() { + unpack_defs(&mut schema, &defs); + } + Some(json!({"type": "json_schema", "schema": filter_output_schema(&schema)})) +} + +/// `_create_json_tool_call_for_response_format`. +fn response_format_tool(schema: &Value) -> Value { + let mut input_schema = Map::from_iter([("type".to_owned(), json!("object"))]); + if let Value::Object(schema) = schema { + input_schema.extend(schema.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + json!({"name": RESPONSE_FORMAT_TOOL_NAME, "input_schema": input_schema}) +} + +/// `cap_thinking_budget_to_max_tokens`. +pub fn cap_thinking_budget(thinking: Value, max_tokens: Option<&Value>) -> Option { + let budget = thinking.get("budget_tokens").and_then(Value::as_i64); + let (Some(max_tokens), Some(budget)) = (max_tokens.and_then(Value::as_f64), budget) else { + return Some(thinking); + }; + if max_tokens <= MIN_THINKING_BUDGET_TOKENS as f64 { + return None; + } + if (budget as f64) < max_tokens { + return Some(thinking); + } + let kind = thinking + .get("type") + .cloned() + .unwrap_or_else(|| json!("enabled")); + Some(json!({"type": kind, "budget_tokens": (max_tokens - 1.0) as i64})) +} + +/// `_map_reasoning_effort`: `None` for `none`. +pub fn map_reasoning_effort( + effort: &str, + caps: &ModelCapabilities<'_>, +) -> Result, Error> { + if effort == "none" { + return Ok(None); + } + if caps.is_adaptive()? { + return Ok(Some(json!({"type": "adaptive", "display": "summarized"}))); + } + let budget = + reasoning_effort_budget(effort).ok_or_else(|| unmapped_reasoning_effort(effort))?; + Ok(Some(json!({"type": "enabled", "budget_tokens": budget}))) +} + +/// `_legacy_budget_to_effort`. +fn legacy_budget_to_effort( + budget: i64, + caps: &ModelCapabilities<'_>, +) -> Result<&'static str, Error> { + Ok( + if budget >= 8192 && caps.supports("supports_xhigh_reasoning_effort")? { + "xhigh" + } else if budget >= 4096 { + "high" + } else if budget >= 2048 { + "medium" + } else { + "low" + }, + ) +} + +/// `translate_legacy_thinking_for_adaptive_model`. +pub fn translate_legacy_thinking( + out: &mut Map, + caps: &ModelCapabilities<'_>, + drop: DropPolicy, +) -> Result<(), Error> { + if !caps.is_adaptive()? || caps.supports("supports_legacy_thinking")? { + return Ok(()); + } + let Some(Value::Object(thinking)) = out.get("thinking").cloned() else { + return Ok(()); + }; + if thinking.get("type").and_then(Value::as_str) != Some("enabled") { + return Ok(()); + } + let budget = match thinking.get("budget_tokens") { + Some(Value::Number(number)) => number.as_f64().unwrap_or(0.0) as i64, + Some(Value::String(text)) => text.trim().parse().unwrap_or(0), + _ => 0, + }; + let effort = legacy_budget_to_effort(budget, caps)?; + let mut rebuilt: Map = thinking + .iter() + .filter(|(key, _)| { + !matches!(key.as_str(), "type" | "budget_tokens" | "display") + && (key.as_str() == "block_binding" || !drop.drops()) + }) + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + rebuilt.insert("type".into(), json!("adaptive")); + if let Some(display) = thinking + .get("display") + .filter(|display| matches!(display.as_str(), Some("summarized" | "omitted"))) + { + rebuilt.insert("display".into(), display.clone()); + } + let mut output_config = Map::from_iter([("effort".to_owned(), json!(effort))]); + if let Some(Value::Object(existing)) = out.get("output_config") { + output_config.extend(existing.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + out.insert("thinking".into(), Value::Object(rebuilt)); + out.insert("output_config".into(), Value::Object(output_config)); + Ok(()) +} + +/// `web_search_options` as Anthropic's hosted web search tool. +pub fn web_search_tool(options: &Map) -> Value { + let mut tool = Map::from_iter([ + ("type".to_owned(), json!("web_search_20250305")), + ("name".to_owned(), json!("web_search")), + ]); + if let Some(location) = options.get("user_location").filter(|v| !v.is_null()) { + let mut user_location = Map::from_iter([("type".to_owned(), json!("approximate"))]); + if let Some(Value::Object(approximate)) = location.get("approximate") { + for (key, value) in approximate { + if matches!(key.as_str(), "city" | "region" | "country" | "timezone") { + user_location.insert(key.clone(), value.clone()); + } + } + } + tool.insert("user_location".into(), Value::Object(user_location)); + } + if let Some(size) = options.get("search_context_size").and_then(Value::as_str) { + let max_uses = match size { + "low" => Some(1), + "medium" => Some(5), + "high" => Some(10), + _ => None, + }; + if let Some(max_uses) = max_uses { + tool.insert("max_uses".into(), json!(max_uses)); + } + } + Value::Object(tool) +} + +/// `map_openai_context_management_to_anthropic`. +pub fn map_context_management(value: &Value) -> Option { + match value { + Value::Object(object) if object.contains_key("edits") => Some(value.clone()), + Value::Array(entries) => { + let edits: Vec = entries + .iter() + .filter_map(Value::as_object) + .filter(|entry| entry.get("type").and_then(Value::as_str) == Some("compaction")) + .map(|entry| { + let mut edit = Map::from_iter([("type".to_owned(), json!("compact_20260112"))]); + if let Some(threshold) = entry.get("compact_threshold").and_then(Value::as_f64) + { + edit.insert( + "trigger".into(), + json!({"type": "input_tokens", "value": threshold as i64}), + ); + } + for (key, value) in entry { + if !matches!(key.as_str(), "type" | "compact_threshold") { + edit.insert(key.clone(), value.clone()); + } + } + Value::Object(edit) + }) + .collect(); + (!edits.is_empty()).then(|| json!({"edits": edits})) + } + _ => None, + } +} + +/// `max_tokens` as Python keeps it: an int as sent, any other number rounded to at least 1. +pub fn max_tokens_value(value: &Value) -> Result { + match value { + Value::Number(number) if number.is_i64() || number.is_u64() => Ok(value.clone()), + Value::Number(number) => { + let rounded = number.as_f64().unwrap_or(1.0).round_ties_even(); + Ok(json!((rounded as i64).max(1))) + } + Value::Bool(flag) => Ok(json!(i64::from(*flag))), + other => Err(Error::InvalidRequest(ErrorDetail::InvalidValue { + field: "max_tokens", + expected: "a number", + actual: other.clone(), + })), + } +} + +/// What `map_openai_params` hands to `transform_request`. +#[derive(Debug, Default)] +pub struct MappedParams { + pub params: Map, + pub json_mode: bool, +} + +/// `get_supported_openai_params`. +pub fn supported_params(caps: &ModelCapabilities<'_>) -> Result, Error> { + let mut params = vec![ + "stream", + "stop", + "temperature", + "top_p", + "max_tokens", + "max_completion_tokens", + "tools", + "tool_choice", + "extra_headers", + "parallel_tool_calls", + "response_format", + "user", + "web_search_options", + "speed", + "context_management", + "cache_control", + ]; + let mut reasoning_keys = caps.candidates.clone(); + reasoning_keys.insert(1, format!("anthropic/{}", caps.model)); + if caps.model.contains("claude-3-7-sonnet") + || caps.is_adaptive()? + || capability(&caps.context, &reasoning_keys, "supports_reasoning")? == Some(true) + { + params.extend(["thinking", "reasoning_effort"]); + } + params.push("service_tier"); + Ok(params) +} + +/// `AnthropicConfig.map_openai_params`. +pub fn map_openai_params( + openai: &Map, + caps: &ModelCapabilities<'_>, + drop: DropPolicy, +) -> Result { + let thinking_enabled = crate::base_llm::chat::params::is_thinking_enabled(openai); + let mut out = Map::new(); + let mut json_mode = false; + let add_tools = |out: &mut Map, tools: Vec| match out.get_mut("tools") { + Some(Value::Array(existing)) => existing.extend(tools), + _ => { + out.insert("tools".into(), Value::Array(tools)); + } + }; + for (name, value) in ordered(openai) { + match name { + "max_tokens" | "max_completion_tokens" => { + out.insert("max_tokens".into(), max_tokens_value(value)?); + } + "tools" => add_tools(&mut out, map_tools(value)?), + "tool_choice" | "parallel_tool_calls" => { + if let Some(choice) = + map_tool_choice(openai.get("tool_choice"), openai.get("parallel_tool_calls")) + { + out.insert( + "tool_choice".into(), + apply_forced_tool_choice(choice, caps, drop)?, + ); + } + } + "stream" if value == &Value::Bool(true) => { + out.insert("stream".into(), json!(true)); + } + "stop" if value.is_string() || value.is_array() => { + if let Some(stop) = map_stop_sequences(value, drop) { + out.insert("stop_sequences".into(), stop); + } + } + "temperature" | "top_p" => { + apply_sampling_param(&mut out, caps, drop, name, value, name)? + } + "response_format" => { + let Some(format) = value.as_object() else { + continue; + }; + if caps.supports("supports_native_structured_output")? { + if let Some(output_format) = output_format(format) { + out.insert("output_format".into(), output_format); + } + } else { + if format.get("type").and_then(Value::as_str) == Some("text") { + continue; + } + let Some(schema) = response_format_schema(format) else { + continue; + }; + if !thinking_enabled && !caps.forced_tool_use_unsupported()? { + out.insert( + "tool_choice".into(), + json!({"name": RESPONSE_FORMAT_TOOL_NAME, "type": "tool"}), + ); + } + add_tools(&mut out, vec![response_format_tool(&schema)]); + } + json_mode = true; + } + "user" => { + if let Some(user) = value.as_str().filter(|user| valid_user_id(user)) { + out.insert("metadata".into(), json!({"user_id": user})); + } + } + "thinking" => { + let adaptive_request = + value.get("type").and_then(Value::as_str) == Some("adaptive"); + if value.is_object() && adaptive_request && !caps.is_adaptive()? { + let max_tokens = openai + .get("max_completion_tokens") + .filter(|v| truthy(v)) + .or_else(|| openai.get("max_tokens")) + .filter(|v| !v.is_null()); + let legacy = map_reasoning_effort("medium", caps)?; + match legacy.and_then(|legacy| cap_thinking_budget(legacy, max_tokens)) { + Some(capped) => { + out.insert("thinking".into(), capped); + } + None => { + out.remove("thinking"); + } + } + } else { + out.insert("thinking".into(), value.clone()); + translate_legacy_thinking(&mut out, caps, drop)?; + } + } + "reasoning_effort" => { + let effort = match value { + Value::Object(effort) => effort.get("effort"), + other => Some(other), + }; + let Some(effort) = effort.and_then(Value::as_str) else { + continue; + }; + match map_reasoning_effort(effort, caps)? { + None => { + out.remove("thinking"); + out.remove("output_config"); + } + Some(thinking) => { + out.insert("thinking".into(), thinking); + if caps.is_adaptive()? { + let mapped = output_config_effort(effort) + .ok_or_else(|| unmapped_reasoning_effort(effort))?; + out.insert("output_config".into(), json!({"effort": mapped})); + } + } + } + } + "web_search_options" => { + if let Some(options) = value.as_object() { + add_tools(&mut out, vec![web_search_tool(options)]); + } + } + "context_management" => { + if let Some(mapped) = map_context_management(value) { + out.insert("context_management".into(), mapped); + } + } + "service_tier" => { + let Some(tier) = value.as_str() else { continue }; + match tier { + "auto" => { + out.insert("service_tier".into(), json!("auto")); + } + "default" | "standard_only" => { + out.insert("service_tier".into(), json!("standard_only")); + } + other => drop.unsupported(|| { + unsupported_value(format!( + "{} does not support service_tier='{other}'. Anthropic accepts 'auto', 'standard_only', or OpenAI's 'default' (sent as 'standard_only'). To drop unsupported params, set `litellm.drop_params = True`.", + caps.model + )) + })?, + } + } + _ => {} + } + } + // `update_optional_params_with_thinking_tokens`. + let thinking_on = out + .get("thinking") + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("enabled"); + if thinking_on + && !openai.contains_key("max_tokens") + && !openai.contains_key("max_completion_tokens") + && let Some(budget) = out + .get("thinking") + .and_then(|thinking| thinking.get("budget_tokens")) + .and_then(Value::as_i64) + { + out.insert( + "max_tokens".into(), + json!(budget + DEFAULT_MAX_TOKENS as i64), + ); + } + Ok(MappedParams { + params: out, + json_mode, + }) +} + +/// The schema of the request's `json_tool_call` tool (`JsonModeRequest.tool_schema`). +pub fn json_tool_schema(tools: Option<&Value>) -> Option { + tools? + .as_array()? + .iter() + .find(|tool| tool.get("name").and_then(Value::as_str) == Some(RESPONSE_FORMAT_TOOL_NAME)) + .and_then(|tool| tool.get("input_schema")) + .and_then(ResponseFormatSchema::from_schema) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + fn object(value: Value) -> Map { + value.as_object().unwrap().clone() + } + + fn caps(model: &str) -> ModelCapabilities<'static> { + ModelCapabilities::new( + model, + &ChatRequestContext { + drop_params: Some(false), + model_info: None, + }, + ) + } + + fn map(params: Value) -> MappedParams { + map_openai_params( + &object(params), + &caps("claude-sonnet-4-5"), + DropPolicy::new(Some(false)), + ) + .unwrap() + } + + #[rstest] + fn candidates_cover_bedrock_ids_and_dotted_versions() { + let candidates = model_map_candidates("us.anthropic.claude-sonnet-4.6-20251101-v1:0"); + assert!(candidates.contains(&"anthropic.claude-sonnet-4.6-20251101-v1:0".to_string())); + assert!(candidates.contains(&"us.anthropic.claude-sonnet-4-6".to_string())); + assert!( + candidates.contains(&"bedrock/anthropic.claude-sonnet-4.6-20251101-v1:0".to_string()) + ); + assert!(model_map_candidates("claude-x@20250101").contains(&"claude-x".to_string())); + } + + #[rstest] + fn tools_and_choices_map_like_python() { + let mapped = map(json!({ + "tools": [{"type": "function", "function": {"name": "f", "description": "d", "parameters": {"type": "object", "properties": {"a": {}}, "title": "x"}}, "cache_control": {"type": "ephemeral"}}], + "tool_choice": "required", + "parallel_tool_calls": false + })); + assert_eq!( + mapped.params, + object(json!({ + "tools": [{"name": "f", "input_schema": {"type": "object", "properties": {"a": {}}}, "type": "custom", "description": "d", "cache_control": {"type": "ephemeral"}}], + "tool_choice": {"type": "any", "disable_parallel_tool_use": true} + })) + ); + assert_eq!( + map(json!({"tool_choice": {"type": "function", "function": {"name": "f"}}})).params["tool_choice"], + json!({"type": "tool", "name": "f"}) + ); + assert_eq!( + map(json!({"parallel_tool_calls": true})).params["tool_choice"], + json!({"type": "auto", "disable_parallel_tool_use": false}) + ); + } + + #[rstest] + fn max_tokens_stop_user_and_sampling() { + let mapped = map( + json!({"max_completion_tokens": 10.6, "stop": "END", "user": "u-1", "temperature": 0.2, "top_p": 0.5}), + ); + assert_eq!( + mapped.params, + object( + json!({"temperature": 0.2, "top_p": 0.5, "stop_sequences": ["END"], "max_tokens": 11, "metadata": {"user_id": "u-1"}}) + ) + ); + assert!( + !map(json!({"user": "a@b.co"})) + .params + .contains_key("metadata") + ); + assert!( + !map(json!({"user": "+1 (555) 123-4567"})) + .params + .contains_key("metadata") + ); + } + + #[rstest] + fn removed_sampling_params_are_refused_or_dropped() { + let refused = map_openai_params( + &object(json!({"temperature": 0.3})), + &caps("claude-opus-4-7"), + DropPolicy::new(Some(false)), + ) + .unwrap_err(); + assert_eq!( + refused.to_string(), + "invalid request: claude-opus-4-7 does not support temperature=0.3. Only temperature=1 is supported. To drop unsupported params, set `litellm.drop_params = True`." + ); + let dropped = map_openai_params( + &object(json!({"temperature": 0.3, "top_p": 0.1})), + &caps("claude-opus-4-7"), + DropPolicy::new(Some(true)), + ) + .unwrap(); + assert!(dropped.params.is_empty()); + } + + #[rstest] + fn response_format_becomes_a_forced_tool() { + let mapped = map( + json!({"response_format": {"type": "json_schema", "json_schema": {"name": "x", "schema": {"type": "object", "properties": {"a": {"type": "string"}}}}}}), + ); + assert!(mapped.json_mode); + assert_eq!( + mapped.params, + object(json!({ + "tool_choice": {"name": "json_tool_call", "type": "tool"}, + "tools": [{"name": "json_tool_call", "input_schema": {"type": "object", "properties": {"a": {"type": "string"}}}}] + })) + ); + assert!(!map(json!({"response_format": {"type": "text"}})).json_mode); + } + + #[rstest] + fn reasoning_effort_sets_a_budget_and_max_tokens() { + let mapped = map(json!({"reasoning_effort": "low"})); + assert_eq!( + mapped.params, + object( + json!({"thinking": {"type": "enabled", "budget_tokens": 1024}, "max_tokens": 5120}) + ) + ); + assert!( + map_openai_params( + &object(json!({"reasoning_effort": "huge"})), + &caps("claude-sonnet-4-5"), + DropPolicy::new(Some(false)) + ) + .is_err() + ); + } + + #[rstest] + fn output_schemas_lose_unenforceable_constraints() { + assert_eq!( + filter_output_schema( + &json!({"type": "object", "properties": {"n": {"type": "integer", "minimum": 1, "description": "count"}, "o": {"oneOf": [{"type": "string"}]}}}) + ), + json!({"type": "object", "properties": {"n": {"description": "count Note: minimum value: 1.", "type": "integer"}, "o": {"anyOf": [{"type": "string"}]}}, "additionalProperties": false}) + ); + } +} diff --git a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs index 49a81adc32e..bbfa16cc344 100644 --- a/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/anthropic/chat/transformation.rs @@ -1,11 +1,15 @@ +//! Python's `AnthropicConfig` for `/chat/completions`: `get_optional_params` and +//! `transform_request` on the way in, `transform_parsed_response` on the way out. + +use std::collections::{BTreeSet, HashMap, HashSet}; + use litellm_auth::SecretValue; -use litellm_core_utils::{ - core_helpers::{finish_reason_for, unix_now, usage_from_parts}, - prompt_templates::factory::{Conversation, build_conversation}, -}; +use litellm_core_utils::core_helpers::{finish_reason_for, unix_now, usage_from_parts}; use litellm_llms_types::{ formats::chat_completions::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, ChatMessage, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, ChatCompletionsChoice, ChatCompletionsChoiceMessage, + ChatCompletionsResponse, ChatMessage, }, providers::anthropic::DEFAULT_HEADERS, }; @@ -13,9 +17,19 @@ use serde::Deserialize; use serde_json::{Map, Value, json}; use crate::{ - Error, + Error, ErrorDetail, anthropic::{ - chat::handler::ModelResponseIterator, + chat::{ + handler::ModelResponseIterator, + messages::{ + anthropic_messages_pt, place_mid_conversation_system, role, + split_leading_system_run, system_blocks, + }, + params::{ + MappedParams, ModelCapabilities, apply_sampling_param, json_tool_schema, + map_openai_params, map_tools, supported_params, valid_user_id, + }, + }, common_utils::{ API_KEY_PLACEMENT, complete_anthropic_url, forwarded_oauth_bearer, resolve_anthropic_api_key, @@ -24,50 +38,39 @@ use crate::{ base_llm::{ auth::AuthScheme, chat::{ + media::{data_url, remote_urls}, + params::{ + ChatRequestContext, DEFAULT_MAX_TOKENS, RESPONSE_FORMAT_TOOL_NAME, capability, + python_dumps, sort_params, + }, + response_format::json_tool_answer, streaming::{ChatStream, StreamShape}, transformation::{ BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, - STREAM_PARAM, Unsupported, ValidatedEnvironment, unsupported_message, - unsupported_param, + STREAM_PARAM, Unsupported, ValidatedEnvironment, }, }, messages::streaming::event_stream as messages_event_stream, }, }; -/// Anthropic parameter names, post `map_openai_params`, that the Rust path can -/// place verbatim in the Messages body. -/// -/// `top_k` is deliberately absent even though the Messages API takes it. -/// `temperature` and `top_p` reach this gate already resolved, because -/// `map_openai_params` runs first and applies `_apply_sampling_param` to them. -/// `top_k` bypasses `map_openai_params` entirely, so Python applies that same -/// per-model gate inside `transform_request`, the function this route replaces. -/// Forwarding it would send `top_k` to a model that removed sampling params and -/// take a 400 after the call, where Python drops it and succeeds. -const SUPPORTED_PARAMS: &[(&str, &str)] = &[ - ("max_tokens", "max_tokens"), - ("temperature", "temperature"), - ("top_p", "top_p"), - ("stop", "stop_sequences"), -]; +pub const BETA_HEADER: &str = "anthropic-beta"; +const TOOL_NAME_MAX_LEN: usize = 128; +const VALID_EFFORTS: [&str; 5] = ["high", "medium", "low", "xhigh", "max"]; #[derive(Deserialize)] -struct TextResponseProjection { +struct ResponseProjection { model: String, - content: Vec, + content: Vec, usage: ResponseUsageProjection, + #[serde(default)] stop_reason: Option, -} - -#[derive(Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum TextResponseBlock { - Text { - text: String, - }, - #[serde(other)] - Other, + #[serde(default)] + stop_details: Option, + #[serde(default)] + context_management: Option, + #[serde(default)] + container: Option, } #[derive(Deserialize)] @@ -84,6 +87,705 @@ pub struct AnthropicConfig; pub const ANTHROPIC_CHAT_COMPLETIONS_CONFIG: AnthropicConfig = AnthropicConfig; +/// `add_dummy_tool`, the tool Anthropic needs when the history holds tool calls and the +/// request names no tools. +fn dummy_tool() -> Value { + json!([{ + "type": "function", + "function": { + "name": "dummy_tool", + "description": "This is a dummy tool call", + "parameters": {"type": "object", "properties": {}}, + }, + }]) +} + +/// `get_prefix_prompt`: the text of a trailing `{"role": "assistant", "prefix": true}`. +fn prefix_prompt(messages: &[Value]) -> Option { + let last = messages.last()?; + let is_prefix = last + .get("prefix") + .is_some_and(crate::base_llm::chat::params::truthy); + (last.get("role").and_then(Value::as_str) == Some("assistant") && is_prefix) + .then(|| { + last.get("content") + .and_then(Value::as_str) + .map(str::to_owned) + }) + .flatten() +} + +/// `_basic_sanitize_anthropic_tool_name`. +fn basic_sanitize_tool_name(name: &str) -> String { + name.chars() + .map( + |c| match c.is_ascii_alphanumeric() || c == '_' || c == '-' { + true => c, + false => '_', + }, + ) + .take(TOOL_NAME_MAX_LEN) + .collect() +} + +/// `_build_anthropic_tool_name_maps`: `(forward, reverse)` for the names that change. +fn tool_name_maps(originals: &[String]) -> (HashMap, HashMap) { + let mut forward: HashMap = HashMap::new(); + let mut used: HashSet = originals + .iter() + .filter(|original| basic_sanitize_tool_name(original) == **original) + .cloned() + .collect(); + for original in originals { + let candidate = basic_sanitize_tool_name(original); + if candidate == *original || forward.contains_key(original) { + continue; + } + let mut unique = candidate.clone(); + let mut n = 1; + while used.contains(&unique) { + n += 1; + let suffix = format!("_{n}"); + let head: String = candidate + .chars() + .take(TOOL_NAME_MAX_LEN - suffix.len()) + .collect(); + unique = format!("{head}{suffix}"); + } + used.insert(unique.clone()); + forward.insert(original.clone(), unique); + } + let reverse = forward + .iter() + .map(|(original, sanitized)| (sanitized.clone(), original.clone())) + .collect(); + (forward, reverse) +} + +/// `_sanitize_tool_names_in_request` and `_rewrite_tool_names_in_messages`. +fn sanitize_tool_names( + params: &mut Map, + messages: &mut [Value], +) -> HashMap { + let Some(Value::Array(tools)) = params.get_mut("tools") else { + return HashMap::new(); + }; + let originals: Vec = tools + .iter() + .filter(|tool| tool.get("type").and_then(Value::as_str) == Some("custom")) + .filter_map(|tool| tool.get("name").and_then(Value::as_str)) + .filter(|name| !name.is_empty()) + .map(str::to_owned) + .collect(); + let (forward, reverse) = tool_name_maps(&originals); + if forward.is_empty() { + return reverse; + } + let rename = |value: Option<&mut Value>| { + if let Some(Value::String(name)) = value + && let Some(sanitized) = forward.get(name.as_str()) + { + *name = sanitized.clone(); + } + }; + for tool in tools.iter_mut() { + if tool.get("type").and_then(Value::as_str) == Some("custom") { + rename(tool.get_mut("name")); + } + } + if let Some(choice) = params.get_mut("tool_choice") + && choice.get("type").and_then(Value::as_str) == Some("tool") + { + rename(choice.get_mut("name")); + } + for message in messages.iter_mut() { + if let Some(Value::Array(calls)) = message.get_mut("tool_calls") { + for call in calls { + rename(call.get_mut("function").and_then(|f| f.get_mut("name"))); + } + } + rename( + message + .get_mut("function_call") + .and_then(|f| f.get_mut("name")), + ); + } + reverse +} + +/// The `anthropic-beta` values the request's features need: the ones Python's +/// `validate_environment` collects and those `update_headers_with_optional_anthropic_beta` +/// adds. Python joins the first lot from a set, so their order carries no meaning. +fn request_betas( + params: &Map, + original_messages: &[Value], + anthropic_messages: &[Value], + caps: &ModelCapabilities<'_>, +) -> Result, Error> { + let tools: Vec<&Value> = params + .get("tools") + .and_then(Value::as_array) + .into_iter() + .flatten() + .collect(); + let mut environment: BTreeSet<&'static str> = BTreeSet::new(); + let tool_field = |tool: &Value, key: &str| -> Option { + tool.get(key) + .or_else(|| tool.get("function").and_then(|f| f.get(key))) + .cloned() + }; + if params + .get("mcp_servers") + .is_some_and(crate::base_llm::chat::params::truthy) + { + environment.insert("mcp-client-2025-04-04"); + } + if tools.iter().any(|tool| { + tool_field(tool, "allowed_callers") + .and_then(|callers| callers.as_array().cloned()) + .is_some_and(|callers| callers.iter().any(|c| c == "code_execution_20250825")) + || tool_field(tool, "input_examples") + .and_then(|examples| examples.as_array().map(|e| !e.is_empty())) + .unwrap_or(false) + }) { + environment.insert("advanced-tool-use-2025-11-20"); + } + let output_config = params.get("output_config"); + if output_config + .and_then(|config| config.get("effort")) + .and_then(Value::as_str) + .is_some_and(|effort| !effort.is_empty()) + && !caps.is_adaptive()? + { + environment.insert("effort-2025-11-24"); + } + if original_messages + .iter() + .any(|message| message.get("output_config").is_some()) + { + environment.insert("mid-conversation-output-config-2026-07-01"); + } + let thinking = params.get("thinking"); + if thinking + .and_then(|t| t.get("type")) + .and_then(Value::as_str) + .is_some_and(|kind| matches!(kind, "adaptive" | "enabled")) + && thinking + .and_then(|t| t.get("display")) + .and_then(Value::as_str) + == Some("updates") + { + environment.insert("thinking-display-updates-2026-08-18"); + } + if original_messages.iter().any(|message| { + role(message) == "system" + && message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|block| { + matches!( + block.get("type").and_then(Value::as_str), + Some("tool_addition" | "tool_removal") + ) && block + .get("tool") + .and_then(|t| t.get("type")) + .and_then(Value::as_str) + == Some("tool_reference") + }) + }) { + environment.insert("mid-conversation-tool-changes-2026-07-01"); + } + + let mut betas: Vec = environment.into_iter().map(str::to_owned).collect(); + let mut add = |beta: &str| { + if !betas.iter().any(|existing| existing == beta) { + betas.push(beta.to_owned()); + } + }; + let signed_compaction = anthropic_messages.iter().any(|message| { + message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .any(|block| { + block.get("type").and_then(Value::as_str) == Some("compaction") + && block + .get("signature") + .and_then(Value::as_str) + .is_some_and(|s| !s.is_empty()) + }) + }); + if params.get("compaction").is_some_and(|v| !v.is_null()) || signed_compaction { + add("compact-2026-09-04"); + } + if let Some(context_management) = params.get("context_management").filter(|v| !v.is_null()) { + let edits = match context_management { + Value::Object(object) if object.contains_key("edits") => { + object.get("edits").and_then(Value::as_array).cloned() + } + Value::Array(entries) => Some(entries.clone()), + _ => None, + }; + if let Some(edits) = edits { + let compact = |edit: &Value| { + matches!( + edit.get("type").and_then(Value::as_str), + Some("compact_20260112" | "compaction") + ) + }; + if edits.iter().any(compact) { + add("compact-2026-01-12"); + } + if edits.iter().any(|edit| !compact(edit)) { + add("context-management-2025-06-27"); + } + } + } + if params.get("output_format").is_some_and(|v| !v.is_null()) + || output_config + .and_then(|config| config.get("format")) + .is_some_and(|v| !v.is_null()) + { + add("structured-outputs-2025-11-13"); + } + if params.get("speed").and_then(Value::as_str) == Some("fast") { + add("fast-mode-2026-02-01"); + } + Ok(betas) +} + +/// `_apply_output_config`: the effort check Python makes before sending `output_config`. +fn apply_output_config( + body: &mut Map, + caps: &ModelCapabilities<'_>, + drops: bool, +) -> Result<(), Error> { + let Some(Value::Object(output_config)) = body.get("output_config").cloned() else { + return Ok(()); + }; + if output_config.is_empty() { + return Ok(()); + } + let supports_effort = || -> Result { + if caps.supports("supports_output_config")? { + return Ok(true); + } + for level in ["low", "minimal", "medium", "high", "xhigh", "max"] { + if caps.supports(&format!("supports_{level}_reasoning_effort"))? { + return Ok(true); + } + } + Ok(false) + }; + if drops && output_config.keys().any(|key| key != "format") && !supports_effort()? { + match output_config.get("format") { + None | Some(Value::Null) => { + body.remove("output_config"); + } + Some(format) => { + body.insert("output_config".into(), json!({"format": format})); + } + } + return Ok(()); + } + let effort = output_config + .get("effort") + .filter(|effort| !effort.is_null()); + if let Some(effort) = effort { + let text = effort.as_str().filter(|text| VALID_EFFORTS.contains(text)); + if text.is_none() { + let shown = match effort { + Value::String(text) => format!("'{text}'"), + other => python_dumps(other), + }; + return Err(Error::InvalidRequest(ErrorDetail::Message(format!( + "Invalid effort value: {shown}. Must be one of: 'high', 'medium', 'low', 'xhigh', 'max'" + )))); + } + let value = match text { + Some("max") + if !(caps.is_adaptive()? || caps.supports("supports_max_reasoning_effort")?) => + { + Some("max") + } + Some("xhigh") if !caps.supports("supports_xhigh_reasoning_effort")? => Some("xhigh"), + _ => None, + }; + if let Some(value) = value { + return Err(Error::InvalidRequest(ErrorDetail::UnsupportedValue { + field: "effort", + value, + model: caps.model.clone(), + })); + } + } + Ok(()) +} + +/// `get_max_tokens_for_model`: the exact cost-map entry's limit, else 4096. +fn default_max_tokens(model: &str, context: &ChatRequestContext<'_>) -> Result { + let Some(info) = context.model_info else { + return match context.drop_params { + None => Err(Error::Unsupported( + "a request whose default max_tokens comes from the model cost map", + )), + Some(_) => Ok(json!(DEFAULT_MAX_TOKENS)), + }; + }; + let limit = info + .entry(model) + .and_then(|entry| match entry.get("max_output_tokens") { + Some(value) => Some(value.clone()), + None => entry.get("max_tokens").cloned(), + }); + Ok(match limit { + Some(value) if value.is_number() => value, + _ => json!(DEFAULT_MAX_TOKENS), + }) +} + +impl AnthropicConfig { + fn build_request( + &self, + model: &str, + messages: Vec, + optional_params: Map, + context: &ChatRequestContext<'_>, + ) -> Result { + let caps = ModelCapabilities::new(model, context); + let request_metadata = optional_params.get("metadata").cloned(); + let sorted = sort_params( + optional_params, + &[], + context, + |_| supported_params(&caps), + "anthropic", + model, + )?; + let drop = sorted.drop; + let MappedParams { + mut params, + json_mode, + } = map_openai_params(&sorted.openai, &caps, drop)?; + for name in &sorted.allowed { + if let Some(value) = sorted.openai.get(name) + && !params.contains_key(name) + { + params.insert(name.clone(), value.clone()); + } + } + params.extend(sorted.provider); + + let mut messages: Vec = messages + .iter() + .map(serde_json::to_value) + .collect::>() + .map_err(|error| Error::InvalidRequest(ErrorDetail::invalid("messages", error)))?; + let original_messages = messages.clone(); + let prefix_prompt = prefix_prompt(&messages); + + if !params.contains_key("tools") + && messages.iter().any(|message| { + message + .get("tool_calls") + .is_some_and(|calls| !calls.is_null()) + }) + { + params.insert("tools".into(), Value::Array(map_tools(&dummy_tool())?)); + } + + // `_maybe_drop_speed_param`: fast mode is the Anthropic API's own, per exact entry. + if let Some(speed) = params.get("speed").cloned() + && capability(context, &[model.to_owned()], "supports_speed")? != Some(true) + { + drop.unsupported(|| { + let shown = match &speed { + Value::String(text) => format!("'{text}'"), + other => python_dumps(other), + }; + Error::InvalidRequest(ErrorDetail::Message(format!( + "{model} does not support speed={shown}. To drop unsupported params, set `litellm.drop_params = True`." + ))) + })?; + params.remove("speed"); + } + + // `maybe_drop_disabled_thinking`. + if params + .get("thinking") + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("disabled") + && caps.supports("thinking_always_on")? + { + params.remove("thinking"); + } + + let tool_name_reverse_map = sanitize_tool_names(&mut params, &mut messages); + + let (leading, later) = split_leading_system_run(messages); + let system = system_blocks(&leading); + if !system.is_empty() { + params.insert("system".into(), Value::Array(system)); + } + let anthropic_messages = anthropic_messages_pt(place_mid_conversation_system(later))?; + + let betas = request_betas(¶ms, &original_messages, &anthropic_messages, &caps)?; + + if !params.contains_key("max_tokens") { + params.insert("max_tokens".into(), default_max_tokens(model, context)?); + } + + if let Some(user_id) = request_metadata + .as_ref() + .and_then(|metadata| metadata.get("user_id")) + .and_then(Value::as_str) + .filter(|user_id| valid_user_id(user_id)) + { + params.insert("metadata".into(), json!({"user_id": user_id})); + } + if let Some(Value::Object(metadata)) = params.get("metadata") { + match metadata.get("user_id").filter(|user_id| !user_id.is_null()) { + Some(user_id) => { + let reduced = json!({"user_id": user_id}); + params.insert("metadata".into(), reduced); + } + None => { + params.remove("metadata"); + } + } + } + params.remove("is_vertex_request"); + params.remove("client_metadata"); + params.remove("json_mode"); + + if let Some(top_k) = params.remove("top_k").filter(|top_k| !top_k.is_null()) { + apply_sampling_param(&mut params, &caps, drop, "top_k", &top_k, "top_k")?; + } + + let speed = params + .get("speed") + .and_then(Value::as_str) + .map(str::to_owned); + let response_format_schema = json_mode + .then(|| json_tool_schema(params.get("tools"))) + .flatten(); + + let mut body = Map::from_iter([ + ("model".to_owned(), json!(model)), + ("messages".to_owned(), Value::Array(anthropic_messages)), + ]); + body.extend(params); + apply_output_config(&mut body, &caps, drop.drops())?; + + Ok(ProviderChatRequestData { + body: Value::Object(body), + stream_shape: StreamShape { + json_mode, + speed, + tool_name_reverse_map, + response_format_schema, + prefix_prompt, + }, + headers: match betas.is_empty() { + true => Vec::new(), + false => vec![(BETA_HEADER.to_owned(), betas.join(", "))], + }, + }) + } +} + +/// `convert_tool_use_to_openai_format`, the name put back the way the caller spelled it. +fn tool_call(block: &Value, index: usize, shape: &StreamShape) -> ChatCompletionToolCallChunk { + let name = block.get("name").and_then(Value::as_str).map(|name| { + shape + .tool_name_reverse_map + .get(name) + .cloned() + .unwrap_or_else(|| name.to_owned()) + }); + ChatCompletionToolCallChunk { + id: block.get("id").and_then(Value::as_str).map(str::to_owned), + tool_type: "function".into(), + function: ChatCompletionToolCallFunctionChunk { + name, + arguments: python_dumps(block.get("input").unwrap_or(&Value::Null)), + provider_specific_fields: None, + }, + index: index as i64, + } +} + +/// Python's `refusal_provider_specific_fields`. +fn refusal_fields(stop_reason: Option<&str>, stop_details: Option<&Value>) -> Map { + let mut fields = Map::new(); + match stop_details.filter(|details| !details.is_null()) { + Some(details) => { + fields.insert("stop_reason".into(), json!(stop_reason)); + fields.insert("stop_details".into(), details.clone()); + } + None if stop_reason == Some("refusal") => { + fields.insert("stop_reason".into(), json!("refusal")); + } + None => {} + } + fields +} + +fn anthropic_response( + body: ResponseProjection, + shape: &StreamShape, +) -> Result { + let mut text = String::new(); + let mut citations: Option> = None; + let mut thinking_blocks: Option> = None; + let mut tool_calls: Vec = Vec::new(); + let mut web_search_results: Option> = None; + let mut tool_results: Option> = None; + let mut compaction_blocks: Option> = None; + for (index, block) in body.content.iter().enumerate() { + let kind = block + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + match kind { + "text" => text.push_str( + block + .get("text") + .and_then(Value::as_str) + .unwrap_or_default(), + ), + "tool_use" | "server_tool_use" => tool_calls.push(tool_call(block, index, shape)), + result if result.ends_with("_tool_result") => { + let list = match result { + "web_search_tool_result" | "web_fetch_tool_result" => &mut web_search_results, + _ => &mut tool_results, + }; + list.get_or_insert_with(Vec::new).push(block.clone()); + } + "thinking" | "redacted_thinking" => thinking_blocks + .get_or_insert_with(Vec::new) + .push(block.clone()), + "compaction" => compaction_blocks + .get_or_insert_with(Vec::new) + .push(block.clone()), + _ => {} + } + if let Some(Value::Array(block_citations)) = block.get("citations") { + let supported_text = block.get("text").cloned().unwrap_or_else(|| json!("")); + citations.get_or_insert_with(Vec::new).push(Value::Array( + block_citations + .iter() + .map(|citation| { + let mut citation = citation.as_object().cloned().unwrap_or_default(); + citation.insert("supported_text".into(), supported_text.clone()); + Value::Object(citation) + }) + .collect(), + )); + } + } + let reasoning_content = thinking_blocks.as_ref().map(|blocks| { + blocks + .iter() + .filter_map(|block| block.get("thinking").and_then(Value::as_str)) + .collect::() + }); + if let Some(prefix) = &shape.prefix_prompt + && !text.starts_with(prefix.as_str()) + { + text = format!("{prefix}{text}"); + } + + // `_build_provider_specific_fields`, unset fields left out. + let mut provider_fields = Map::new(); + let mut put = |key: &str, value: Option| { + if let Some(value) = value.filter(|value| !value.is_null()) { + provider_fields.insert(key.to_owned(), value); + } + }; + put("citations", citations.map(Value::Array)); + put("thinking_blocks", thinking_blocks.clone().map(Value::Array)); + put("context_management", body.context_management.clone()); + put("web_search_results", web_search_results.map(Value::Array)); + put("tool_results", tool_results.map(Value::Array)); + put("container", body.container.clone()); + put("compaction_blocks", compaction_blocks.map(Value::Array)); + let refusal = refusal_fields(body.stop_reason.as_deref(), body.stop_details.as_ref()); + provider_fields.extend(refusal.clone()); + + let thinking_blocks: Option> = thinking_blocks + .map(|blocks| serde_json::from_value(Value::Array(blocks))) + .transpose() + .map_err(|error| Error::InvalidResponse(ErrorDetail::invalid("thinking block", error)))?; + + let mut stop_reason = body.stop_reason.clone().unwrap_or_default(); + let json_indices: Vec = match shape.json_mode { + true => tool_calls + .iter() + .enumerate() + .filter(|(_, call)| call.function.name.as_deref() == Some(RESPONSE_FORMAT_TOOL_NAME)) + .map(|(index, _)| index) + .collect(), + false => Vec::new(), + }; + let schema = shape.response_format_schema.as_ref(); + let message = if !json_indices.is_empty() && json_indices.len() == tool_calls.len() { + stop_reason = "stop".into(); + let arguments: Vec = tool_calls + .iter() + .map(|call| call.function.arguments.clone()) + .collect(); + ChatCompletionsChoiceMessage { + role: "assistant".into(), + content: Some(json_tool_answer(&arguments, schema)), + provider_specific_fields: (!refusal.is_empty()).then_some(refusal), + ..ChatCompletionsChoiceMessage::default() + } + } else { + if !json_indices.is_empty() { + let arguments: Vec = json_indices + .iter() + .map(|index| tool_calls[*index].function.arguments.clone()) + .collect(); + text.push_str(&json_tool_answer(&arguments, schema)); + let mut position = 0; + tool_calls.retain(|_| { + let keep = !json_indices.contains(&position); + position += 1; + keep + }); + } + ChatCompletionsChoiceMessage { + role: "assistant".into(), + content: (!text.is_empty()).then_some(text), + tool_calls: (!tool_calls.is_empty()).then_some(tool_calls), + reasoning_content, + thinking_blocks, + provider_specific_fields: (!provider_fields.is_empty()).then_some(provider_fields), + } + }; + + Ok(ChatCompletionsResponse { + created: unix_now(), + model: body.model, + choices: vec![ChatCompletionsChoice { + index: 0, + message, + finish_reason: finish_reason_for(&stop_reason).to_string(), + }], + usage: usage_from_parts( + body.usage.input_tokens, + body.usage.output_tokens, + body.usage.cache_read_input_tokens.unwrap_or(0), + body.usage.cache_creation_input_tokens.unwrap_or(0), + ), + }) +} + impl BaseConfig for AnthropicConfig { fn secret_names(&self) -> Vec<&'static str> { use crate::anthropic::common_utils::{ @@ -96,10 +798,6 @@ impl BaseConfig for AnthropicConfig { ] } - fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)] { - SUPPORTED_PARAMS - } - fn get_complete_url( &self, api_base: Option<&str>, @@ -115,59 +813,27 @@ impl BaseConfig for AnthropicConfig { model: &str, messages: Vec, optional_params: Map, + context: &ChatRequestContext<'_>, ) -> Result { - Ok(ProviderChatRequestData { - body: anthropic_body(model, &build_conversation(&messages), optional_params), - stream_shape: StreamShape::default(), - }) + self.build_request(model, messages, optional_params, context) } fn transform_response( &self, _model: &str, response: ProviderChatResponseData, + shape: &StreamShape, ) -> Result { - let body: TextResponseProjection = - serde_json::from_value(response.body).map_err(|error| { - Error::InvalidResponse(crate::ErrorDetail::invalid("messages response", error)) - })?; - // The route declines tool and thinking requests, so a non-text block - // means the response carries something this path never asked for. - // Decline rather than silently dropping it; the host falls back. - if body - .content - .iter() - .any(|block| matches!(block, TextResponseBlock::Other)) - { - return Err(Error::Unsupported("non-text response content block")); - } - let text: String = body - .content - .into_iter() - .map(|block| match block { - TextResponseBlock::Text { text } => text, - TextResponseBlock::Other => String::new(), - }) - .collect(); + let body: ResponseProjection = serde_json::from_value(response.body).map_err(|error| { + Error::InvalidResponse(ErrorDetail::invalid("messages response", error)) + })?; + anthropic_response(body, shape) + } - Ok(ChatCompletionsResponse { - created: unix_now(), - model: body.model, - choices: vec![ChatCompletionsChoice { - index: 0, - message: ChatCompletionsChoiceMessage { - role: "assistant".to_string(), - content: (!text.is_empty()).then_some(text), - }, - finish_reason: finish_reason_for(body.stop_reason.as_deref().unwrap_or("")) - .to_string(), - }], - usage: usage_from_parts( - body.usage.input_tokens, - body.usage.output_tokens, - body.usage.cache_read_input_tokens.unwrap_or(0), - body.usage.cache_creation_input_tokens.unwrap_or(0), - ), + /// `inlines_remote_media`: Anthropic fetches an `https` image itself, not an `http` one. + fn remote_media(&self, messages: &[ChatMessage]) -> Vec { + remote_urls(messages, &["user"], false, |url| { + url.starts_with("http://") && data_url(url).is_none() }) } @@ -214,107 +880,105 @@ impl BaseConfig for AnthropicConfig { DEFAULT_HEADERS } - /// An OAuth bearer is the whole credential: Python's `validate_environment` - /// authenticates with it and drops `x-api-key` rather than resolving one, so - /// the resolved key must not be applied over the top. Any other forwarded - /// `authorization` is unrelated to this header and does not defer, which is - /// also what Python does: it sends the deployment's `x-api-key` alongside. + /// Every request is translated; a param or message Python refuses is refused here too. fn unsupported_reason( &self, - messages: &[ChatMessage], - optional_params: &Map, + _messages: &[ChatMessage], + _optional_params: &Map, ) -> Option { - unsupported_param(self.supported_openai_param_mappings(), &[], optional_params) - .or_else(|| messages.iter().find_map(unsupported_message)) - // Anthropic rejects a request whose first turn is not a user turn. - // Python only repairs that under `litellm.modify_params`, which the - // core cannot observe, so decline instead of guessing. - .or_else(|| { - (!build_conversation(messages).opens_on_user_turn()) - .then_some(Unsupported("conversation does not open on a user turn")) - }) + None } } -fn text_block(text: &str) -> Value { - json!({"type": "text", "text": text}) -} - -fn anthropic_body( - model: &str, - conversation: &Conversation, - optional_params: Map, -) -> Value { - let messages: Vec = conversation - .turns - .iter() - .map(|turn| { - json!({ - "role": <&'static str>::from(turn.role), - "content": turn.texts.iter().map(|text| text_block(text)).collect::>(), - }) - }) - .collect(); - - let system: Vec = conversation.system.iter().map(|s| text_block(s)).collect(); - - let body = Map::from_iter( - [ - ("model".to_string(), json!(model)), - ("messages".to_string(), json!(messages)), - ] - .into_iter() - // Python builds `{"model", "messages", **optional_params}` with - // `system` already folded into optional_params, so a caller-supplied - // key of the same name wins here too. - .chain((!system.is_empty()).then(|| ("system".to_string(), json!(system)))) - .chain(optional_params), - ); - Value::Object(body) -} - #[cfg(test)] mod tests { - use crate::{ - Error, - anthropic::chat::transformation::ANTHROPIC_CHAT_COMPLETIONS_CONFIG, - base_llm::{ - auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, - }, - }; + use std::collections::HashMap; + use litellm_llms_types::formats::chat_completions::{ChatCompletionsResponse, ChatMessage}; use rstest::rstest; use serde_json::{Map, Value, json}; + use super::*; + use crate::base_llm::model_info::ModelInfo; + + struct Map1(HashMap>); + + impl ModelInfo for Map1 { + fn entry(&self, model: &str) -> Option>> { + self.0.get(model).map(std::borrow::Cow::Borrowed) + } + fn declares(&self, key: &str) -> bool { + self.0.values().any(|entry| entry.contains_key(key)) + } + } + + fn cost_map() -> Map1 { + let entry = |value: Value| value.as_object().cloned().unwrap(); + Map1(HashMap::from([ + ( + "claude-sonnet-4-5".to_owned(), + entry(json!({"max_output_tokens": 64000, "supports_reasoning": true})), + ), + ( + "claude-opus-4-8".to_owned(), + entry( + json!({"max_output_tokens": 128000, "supports_adaptive_thinking": true, + "supports_speed": true, "supports_sampling_params": false, + "supports_native_structured_output": true}), + ), + ), + ])) + } + fn messages(value: Value) -> Vec { serde_json::from_value(value).expect("valid messages") } fn params(value: Value) -> Map { - match value { - Value::Object(map) => map, - other => panic!("params must be an object, got {other}"), - } + value.as_object().cloned().expect("params object") + } + + fn request( + model: &str, + msgs: Value, + opts: Value, + drop: bool, + ) -> Result { + let info = cost_map(); + let context = ChatRequestContext { + drop_params: Some(drop), + model_info: Some(&info), + }; + ANTHROPIC_CHAT_COMPLETIONS_CONFIG.transform_request( + model, + messages(msgs), + params(opts), + &context, + ) } fn transform(model: &str, msgs: Value, opts: Value) -> Value { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG - .transform_request(model, messages(msgs), params(opts)) + request(model, msgs, opts, false) .expect("request transforms") .body } fn transform_response(body: Value) -> Result { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG - .transform_response("claude-sonnet-4-5", ProviderChatResponseData { body }) + transform_response_with(body, &StreamShape::default()) } - fn reason(msgs: Value, opts: Value) -> Option { - ANTHROPIC_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + fn transform_response_with( + body: Value, + shape: &StreamShape, + ) -> Result { + ANTHROPIC_CHAT_COMPLETIONS_CONFIG.transform_response( + "claude-sonnet-4-5", + ProviderChatResponseData { body }, + shape, + ) } - #[test] + #[rstest] fn builds_the_messages_body_python_builds() { let body = transform( "claude-sonnet-4-5", @@ -322,7 +986,7 @@ mod tests { {"role": "system", "content": "be terse"}, {"role": "user", "content": "hi"} ]), - json!({"max_tokens": 128, "temperature": 0.2}), + json!({"max_tokens": 128, "temperature": 0.2, "stop": "END", "user": "u-1"}), ); assert_eq!( body, @@ -331,354 +995,466 @@ mod tests { "messages": [ {"role": "user", "content": [{"type": "text", "text": "hi"}]} ], - "system": [{"type": "text", "text": "be terse"}], + "temperature": 0.2, + "stop_sequences": ["END"], "max_tokens": 128, - "temperature": 0.2 + "metadata": {"user_id": "u-1"}, + "system": [{"type": "text", "text": "be terse"}] }) ); } - #[test] - fn omits_system_when_no_system_message_is_present() { + #[rstest] + #[case::from_the_cost_map("claude-sonnet-4-5", 64000)] + #[case::unknown_model("claude-unknown", 4096)] + fn max_tokens_defaults_like_python(#[case] model: &str, #[case] expected: u64) { + let body = transform(model, json!([{"role": "user", "content": "hi"}]), json!({})); + assert_eq!(body["max_tokens"], json!(expected)); let body = transform( - "claude-sonnet-4-5", + model, json!([{"role": "user", "content": "hi"}]), - json!({"max_tokens": 16}), + json!({"max_completion_tokens": 7}), ); - assert!(body.get("system").is_none()); + assert_eq!(body["max_tokens"], json!(7)); } - #[test] - fn merges_consecutive_turns_and_wraps_every_text_in_a_block() { + #[rstest] + fn tools_tool_choice_and_history_round_trip() { let body = transform( "claude-sonnet-4-5", json!([ - {"role": "user", "content": "one"}, - {"role": "user", "content": [{"type": "text", "text": "two"}]}, - {"role": "assistant", "content": "ack"} - ]), - json!({"max_tokens": 16}), - ); - assert_eq!( - body["messages"], - json!([ - {"role": "user", "content": [ - {"type": "text", "text": "one"}, - {"type": "text", "text": "two"} + {"role": "user", "content": "weather?"}, + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_1", "type": "function", + "function": {"name": "get.weather", "arguments": "{\"city\":\"Paris\"}"}} ]}, - {"role": "assistant", "content": [{"type": "text", "text": "ack"}]} - ]) - ); - } - - #[test] - fn right_strips_a_trailing_assistant_prefill_like_python() { - let body = transform( - "claude-sonnet-4-5", - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "Argentina "} + {"role": "tool", "tool_call_id": "call_1", "content": "sunny"} ]), - json!({"max_tokens": 16}), - ); - assert_eq!( - body["messages"][1]["content"][0]["text"], - json!("Argentina") - ); - } - - #[test] - fn passes_every_supported_param_through_untouched() { - let body = transform( - "claude-sonnet-4-5", - json!([{"role": "user", "content": "hi"}]), json!({ - "max_tokens": 64, - "temperature": 0.1, - "top_p": 0.9, - "stop_sequences": ["STOP"] + "max_tokens": 10, + "tools": [{"type": "function", "function": { + "name": "get.weather", "description": "d", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}} + }}], + "tool_choice": {"type": "function", "function": {"name": "get.weather"}}, + "parallel_tool_calls": false }), ); - assert_eq!(body["max_tokens"], json!(64)); - assert_eq!(body["temperature"], json!(0.1)); - assert_eq!(body["top_p"], json!(0.9)); - assert_eq!(body["stop_sequences"], json!(["STOP"])); - } - - #[test] - fn declines_top_k_because_python_gates_it_by_model_below_this_point() { - // `temperature` and `top_p` arrive already resolved, because - // `map_openai_params` applies `_apply_sampling_param` to them before the - // gate runs. `top_k` bypasses that and is gated inside `transform_request`, - // the function this route replaces, so forwarding it would send `top_k` to - // a model that removed sampling params and take a 400 after the call, where - // Python drops it and succeeds. assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"top_k": 40}) - ), - Some(Unsupported("unrecognized request parameter")) + body["tools"], + json!([{"name": "get_weather", "input_schema": {"type": "object", "properties": {"city": {"type": "string"}}}, "type": "custom", "description": "d"}]) ); - } - - #[test] - fn declines_streaming_before_anything_else() { assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": true, "max_tokens": 16}) - ), - Some(Unsupported("streaming")) + body["tool_choice"], + json!({"type": "tool", "name": "get_weather", "disable_parallel_tool_use": true}) ); - } - - #[test] - fn accepts_an_explicit_stream_false() { assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": false, "max_tokens": 16}) - ), - None + body["messages"][1], + json!({"role": "assistant", "content": [ + {"type": "tool_use", "id": "call_1", "name": "get_weather", "input": {"city": "Paris"}} + ]}) + ); + assert_eq!( + body["messages"][2], + json!({"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "call_1", "content": "sunny"} + ]}) ); } #[rstest] - #[case::tools(json!({"tools": []}))] - #[case::tool_choice(json!({"tool_choice": {"type": "auto"}}))] - #[case::thinking(json!({"thinking": {"type": "enabled"}}))] - #[case::system(json!({"system": "injected"}))] - #[case::metadata(json!({"metadata": {"user_id": "u1"}}))] - #[case::output_config(json!({"output_config": {"effort": "high"}}))] - fn declines_any_param_outside_the_allowlist(#[case] param: Value) { + fn tool_history_without_tools_gets_the_dummy_tool() { + let body = transform( + "claude-sonnet-4-5", + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "tool_calls": [ + {"id": "c", "type": "function", "function": {"name": "f", "arguments": ""}} + ]}, + {"role": "tool", "tool_call_id": "c", "content": "ok"} + ]), + json!({"max_tokens": 5}), + ); + assert_eq!(body["tools"][0]["name"], "dummy_tool"); + assert_eq!(body["tools"][0]["description"], "This is a dummy tool call"); + } + + #[rstest] + fn response_format_becomes_a_forced_json_tool() { + let data = request( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"max_tokens": 5, "response_format": {"type": "json_schema", "json_schema": { + "name": "x", "schema": {"type": "object", "properties": {"a": {"type": "string"}}} + }}}), + false, + ) + .unwrap(); assert_eq!( - reason(json!([{"role": "user", "content": "hi"}]), param.clone()), - Some(Unsupported("unrecognized request parameter")), - "expected {param} to decline" + data.body["tool_choice"], + json!({"name": "json_tool_call", "type": "tool"}) + ); + assert_eq!(data.body["tools"][0]["name"], "json_tool_call"); + assert!(data.body.get("json_mode").is_none()); + assert!(data.stream_shape.json_mode); + assert!(data.stream_shape.response_format_schema.is_some()); + } + + #[rstest] + fn native_structured_output_needs_the_beta() { + let data = request( + "claude-opus-4-8", + json!([{"role": "user", "content": "hi"}]), + json!({"max_tokens": 5, "response_format": {"type": "json_schema", "json_schema": { + "name": "x", "schema": {"type": "object", "properties": {"a": {"type": "string"}}} + }}}), + false, + ) + .unwrap(); + assert!(data.body.get("output_format").is_some()); + assert_eq!( + data.headers, + vec![( + "anthropic-beta".to_owned(), + "structured-outputs-2025-11-13".to_owned() + )] ); } - #[test] - fn declines_tool_calls_tool_results_and_multimodal_content() { + #[rstest] + fn unsupported_params_are_refused_or_dropped() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let error = request("claude-sonnet-4-5", msgs.clone(), json!({"seed": 1}), false) + .map(|_| ()) + .unwrap_err(); + assert!(matches!( + error, + Error::InvalidRequest(ErrorDetail::UnsupportedParams { ref params, .. }) if params == &["seed"] + )); + let body = request("claude-sonnet-4-5", msgs.clone(), json!({"seed": 1}), true) + .unwrap() + .body; + assert!(body.get("seed").is_none()); + let body = request( + "claude-sonnet-4-5", + msgs.clone(), + json!({"seed": 1, "drop_params": true}), + false, + ) + .unwrap() + .body; + assert!(body.get("seed").is_none()); + let body = request( + "claude-sonnet-4-5", + msgs.clone(), + json!({"seed": 1, "allowed_openai_params": ["seed"]}), + false, + ) + .unwrap() + .body; + assert_eq!(body["seed"], 1); + let body = request( + "claude-sonnet-4-5", + msgs, + json!({"temperature": 0.5, "additional_drop_params": ["temperature"]}), + false, + ) + .unwrap() + .body; + assert!(body.get("temperature").is_none()); + } + + #[rstest] + fn n_above_one_is_refused() { + let error = request( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"n": 2}), + false, + ) + .map(|_| ()) + .unwrap_err(); + assert!(matches!( + error, + Error::InvalidRequest(ErrorDetail::UnsupportedParams { .. }) + )); + } + + #[rstest] + fn sampling_params_follow_the_model() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let error = request("claude-opus-4-8", msgs.clone(), json!({"top_k": 3}), false) + .map(|_| ()) + .unwrap_err(); assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "c1", "type": "function", - "function": {"name": "f", "arguments": "{}"}} - ]} - ]), - json!({}) - ), - Some(Unsupported("unrecognized message field")) + error.to_string(), + "invalid request: claude-opus-4-8 does not support top_k=3. To drop unsupported params, set `litellm.drop_params = True`." ); + let body = request("claude-opus-4-8", msgs.clone(), json!({"top_k": 3}), true) + .unwrap() + .body; + assert!(body.get("top_k").is_none()); + let body = transform("claude-sonnet-4-5", msgs, json!({"top_k": 3})); + assert_eq!(body["top_k"], 3); + } + + #[rstest] + fn speed_is_checked_against_the_exact_entry() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let data = request( + "claude-opus-4-8", + msgs.clone(), + json!({"speed": "fast"}), + false, + ) + .unwrap(); + assert_eq!(data.body["speed"], "fast"); + assert_eq!(data.stream_shape.speed.as_deref(), Some("fast")); + assert_eq!(data.headers[0].1, "fast-mode-2026-02-01"); + let error = request("claude-sonnet-4-5", msgs, json!({"speed": "fast"}), false) + .map(|_| ()) + .unwrap_err(); assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "tool", "tool_call_id": "c1", "content": "ok"} - ]), - json!({}) - ), - Some(Unsupported("unrecognized message field")) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": [ - {"type": "image_url", "image_url": {"url": "https://x/y.png"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": [ - {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) + error.to_string(), + "invalid request: claude-sonnet-4-5 does not support speed='fast'. To drop unsupported params, set `litellm.drop_params = True`." ); } - #[test] - fn declines_a_message_whose_content_list_is_empty() { - // An empty list passes every per-part check, so without this it would reach - // the provider as an empty `content` array and fail after the call rather - // than declining to Python before it. - assert_eq!( - reason(json!([{"role": "user", "content": []}]), json!({})), - Some(Unsupported("message without content")) + #[rstest] + fn reasoning_effort_maps_to_thinking() { + let body = transform( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"reasoning_effort": "low"}), ); assert_eq!( - reason( - json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]), - json!({}) - ), - None + body["thinking"], + json!({"type": "enabled", "budget_tokens": 1024}) + ); + assert_eq!(body["max_tokens"], json!(1024 + 4096)); + } + + #[rstest] + fn output_config_effort_is_validated() { + let error = request( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"output_config": {"effort": "max"}}), + false, + ) + .map(|_| ()) + .unwrap_err(); + assert_eq!( + error.to_string(), + "invalid request: effort='max' is not supported by this model. Got model: claude-sonnet-4-5" + ); + let error = request( + "claude-sonnet-4-5", + json!([{"role": "user", "content": "hi"}]), + json!({"output_config": {"effort": "huge"}}), + false, + ) + .map(|_| ()) + .unwrap_err(); + assert!(error.to_string().contains("Invalid effort value: 'huge'")); + } + + #[rstest] + fn metadata_keeps_only_a_user_id() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let body = transform( + "claude-sonnet-4-5", + msgs.clone(), + json!({"metadata": {"user_id": "abc", "tags": ["x"]}}), + ); + assert_eq!(body["metadata"], json!({"user_id": "abc"})); + let body = transform("claude-sonnet-4-5", msgs, json!({"user": "a@b.com"})); + assert!(body.get("metadata").is_none()); + } + + #[rstest] + fn without_a_known_drop_policy_refusals_go_back_to_python() { + let context = ChatRequestContext::default(); + let error = ANTHROPIC_CHAT_COMPLETIONS_CONFIG + .transform_request( + "claude-sonnet-4-5", + messages(json!([{"role": "user", "content": "hi"}])), + params(json!({"max_tokens": 3})), + &context, + ) + .map(|_| ()) + .unwrap_err(); + assert!(matches!(error, Error::Unsupported(_))); + } + + #[rstest] + fn only_http_images_are_downloaded() { + let urls = ANTHROPIC_CHAT_COMPLETIONS_CONFIG.remote_media(&messages(json!([ + {"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "http://x/a.png"}}, + {"type": "image_url", "image_url": {"url": "https://x/b.png"}} + ]} + ]))); + assert_eq!(urls, vec!["http://x/a.png"]); + } + + #[rstest] + fn a_data_url_image_becomes_a_base64_block() { + let body = transform( + "claude-sonnet-4-5", + json!([{"role": "user", "content": [ + {"type": "text", "text": "what is this"}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgo="}} + ]}]), + json!({"max_tokens": 5}), + ); + assert_eq!( + body["messages"][0]["content"][1], + json!({"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgo="}}) ); } - #[test] - fn declines_a_conversation_that_does_not_open_on_a_user_turn() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "assistant", "content": "prefill"} - ]), - json!({}) - ), - Some(Unsupported("conversation does not open on a user turn")) - ); - } - - #[test] - fn accepts_a_plain_text_conversation() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "hello"}, - {"role": "user", "content": [{"type": "text", "text": "again"}]} - ]), - json!({"max_tokens": 16, "temperature": 0.5}) - ), - None - ); - } - - #[test] + #[rstest] fn normalizes_a_text_response_into_openai_shape() { let response = transform_response(json!({ "id": "msg_123", - "type": "message", - "role": "assistant", "model": "claude-sonnet-4-5-20260101", "content": [{"type": "text", "text": "hello"}, {"type": "text", "text": " there"}], "stop_reason": "end_turn", - "stop_sequence": null, "usage": {"input_tokens": 11, "output_tokens": 4} })) .expect("response transforms"); - assert_eq!(response.model, "claude-sonnet-4-5-20260101"); - assert_eq!(response.choices.len(), 1); - assert_eq!(response.choices[0].index, 0); - assert_eq!(response.choices[0].message.role, "assistant"); assert_eq!( response.choices[0].message.content.as_deref(), Some("hello there") ); assert_eq!(response.choices[0].finish_reason, "stop"); - assert_eq!(response.usage.prompt_tokens, 11); - assert_eq!(response.usage.completion_tokens, 4); assert_eq!(response.usage.total_tokens, 15); + let value = serde_json::to_value(response).unwrap(); + assert!(value.get("id").is_none()); + assert!(value["choices"][0]["message"].get("tool_calls").is_none()); } - #[test] + #[rstest] fn folds_cache_tokens_into_prompt_tokens_like_python() { let response = transform_response(json!({ "model": "claude-sonnet-4-5", "content": [{"type": "text", "text": "hi"}], "stop_reason": "end_turn", - "usage": { - "input_tokens": 10, - "output_tokens": 2, - "cache_read_input_tokens": 5, - "cache_creation_input_tokens": 3 - } + "usage": {"input_tokens": 10, "output_tokens": 2, + "cache_read_input_tokens": 5, "cache_creation_input_tokens": 3} })) .expect("response transforms"); assert_eq!(response.usage.prompt_tokens, 18); - assert_eq!(response.usage.total_tokens, 20); assert_eq!(response.usage.prompt_tokens_details.cached_tokens, 5); assert_eq!( response.usage.prompt_tokens_details.cache_creation_tokens, 3 ); - assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10); } - #[test] - fn maps_max_tokens_stop_reason_to_length() { + #[rstest] + #[case::refusal("refusal", "content_filter")] + #[case::max_tokens("max_tokens", "length")] + #[case::tool_use("tool_use", "tool_calls")] + fn maps_stop_reasons(#[case] stop_reason: &str, #[case] expected: &str) { let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "max_tokens", + "model": "m", "content": [], "stop_reason": stop_reason, "usage": {"input_tokens": 1, "output_tokens": 1} })) - .expect("response transforms"); - assert_eq!(response.choices[0].finish_reason, "length"); - } - - #[test] - fn a_refusal_returns_the_completion_python_returns() { - // `refusal` is a stop_reason, not a content block type, so the content is - // ordinary text and this normalizes rather than declining. Python maps it - // to content_filter in _FINISH_REASON_MAP and returns the completion. - let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "I can't help with that."}], - "stop_reason": "refusal", - "usage": {"input_tokens": 9, "output_tokens": 6} - })) - .expect("a refusal still transforms"); - assert_eq!(response.choices[0].finish_reason, "content_filter"); - assert_eq!( - response.choices[0].message.content.as_deref(), - Some("I can't help with that.") - ); - } - - #[test] - fn reports_no_content_rather_than_an_empty_string() { - let response = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [], - "stop_reason": "end_turn", - "usage": {"input_tokens": 1, "output_tokens": 0} - })) - .expect("response transforms"); + .unwrap(); + assert_eq!(response.choices[0].finish_reason, expected); assert_eq!(response.choices[0].message.content, None); } - #[test] - fn response_carries_no_id_so_python_keeps_its_chatcmpl_id() { - let response = transform_response(json!({ - "id": "msg_should_not_leak", - "model": "claude-sonnet-4-5", - "content": [{"type": "text", "text": "hi"}], - "stop_reason": "end_turn", - "usage": {"input_tokens": 1, "output_tokens": 1} - })) - .expect("response transforms"); - let value = serde_json::to_value(response).expect("serializable"); - assert!( - value.get("id").is_none(), - "the rust response must not carry an id, got {value}" + #[rstest] + fn tool_use_and_thinking_come_back_as_openai_fields() { + let shape = StreamShape { + tool_name_reverse_map: HashMap::from([("get_weather".into(), "get.weather".into())]), + ..StreamShape::default() + }; + let response = transform_response_with( + json!({ + "model": "m", + "content": [ + {"type": "thinking", "thinking": "hmm", "signature": "sig"}, + {"type": "text", "text": "checking"}, + {"type": "tool_use", "id": "toolu_1", "name": "get_weather", "input": {"city": "Paris"}} + ], + "stop_reason": "tool_use", + "usage": {"input_tokens": 1, "output_tokens": 1} + }), + &shape, + ) + .unwrap(); + let message = serde_json::to_value(&response.choices[0].message).unwrap(); + assert_eq!( + message["tool_calls"], + json!([{"id": "toolu_1", "type": "function", "function": {"name": "get.weather", "arguments": "{\"city\": \"Paris\"}"}, "index": 2}]) + ); + assert_eq!(message["reasoning_content"], "hmm"); + assert_eq!( + message["thinking_blocks"], + json!([{"type": "thinking", "thinking": "hmm", "signature": "sig"}]) + ); + assert_eq!(message["content"], "checking"); + assert_eq!(response.choices[0].finish_reason, "tool_calls"); + } + + #[rstest] + fn a_json_tool_answer_becomes_the_content() { + let shape = StreamShape { + json_mode: true, + ..StreamShape::default() + }; + let response = transform_response_with( + json!({ + "model": "m", + "content": [{"type": "tool_use", "id": "t", "name": "json_tool_call", "input": {"a": "b"}}], + "stop_reason": "tool_use", + "usage": {"input_tokens": 1, "output_tokens": 1} + }), + &shape, + ) + .unwrap(); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("{\"a\": \"b\"}") + ); + assert_eq!(response.choices[0].message.tool_calls, None); + assert_eq!(response.choices[0].finish_reason, "stop"); + } + + #[rstest] + fn a_prefix_prompt_is_put_back() { + let data = request( + "claude-sonnet-4-5", + json!([ + {"role": "user", "content": "hi"}, + {"role": "assistant", "content": "Argentina", "prefix": true} + ]), + json!({"max_tokens": 5}), + false, + ) + .unwrap(); + assert_eq!( + data.stream_shape.prefix_prompt.as_deref(), + Some("Argentina") + ); + let response = transform_response_with( + json!({"model": "m", "content": [{"type": "text", "text": " is"}], + "stop_reason": "end_turn", "usage": {"input_tokens": 1, "output_tokens": 1}}), + &data.stream_shape, + ) + .unwrap(); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("Argentina is") ); } - #[test] - fn declines_a_response_carrying_a_non_text_block() { - let err = transform_response(json!({ - "model": "claude-sonnet-4-5", - "content": [{"type": "tool_use", "id": "t1", "name": "f", "input": {}}], - "stop_reason": "tool_use", - "usage": {"input_tokens": 1, "output_tokens": 1} - })) - .expect_err("non-text block"); - assert_eq!(err, Error::Unsupported("non-text response content block")); - } - - #[rstest::rstest] + #[rstest] #[case::not_an_object(json!("nope"))] #[case::missing_content(json!({"model": "test-model", "usage": {"input_tokens": 1, "output_tokens": 1}}))] #[case::missing_usage(json!({"model": "test-model", "content": []}))] @@ -697,7 +1473,7 @@ mod tests { ); } - #[test] + #[rstest] fn resolves_the_messages_url_and_x_api_key_auth() { let config = &ANTHROPIC_CHAT_COMPLETIONS_CONFIG; assert_eq!( @@ -730,4 +1506,14 @@ mod tests { ] ); } + + #[rstest] + fn tool_names_are_sanitized_without_collisions() { + let (forward, reverse) = + tool_name_maps(&["a.b".to_owned(), "a_b".to_owned(), "c d".to_owned()]); + assert_eq!(forward["a.b"], "a_b_2"); + assert_eq!(forward["c d"], "c_d"); + assert!(!forward.contains_key("a_b")); + assert_eq!(reverse["a_b_2"], "a.b"); + } } diff --git a/litellm-rust/crates/llms/src/azure/chat/transformation.rs b/litellm-rust/crates/llms/src/azure/chat/transformation.rs index 7c6c1056f10..38df96b20cf 100644 --- a/litellm-rust/crates/llms/src/azure/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/azure/chat/transformation.rs @@ -8,56 +8,15 @@ use serde_json::{Map, Value}; use crate::{ Error, ErrorDetail, - base_llm::model_info::{self, ModelInfo}, + base_llm::{ + chat::params::{ + OPENAI_PARAMS, UNVALIDATED_PARAMS, ordered, request_drop_params, string_list, truthy, + }, + model_info::{self, ModelInfo}, + }, openai::chat::transformation::{is_litellm_owned, take_extra_body}, }; -/// The keys of `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` a request carries as OpenAI params, in -/// that table's order. Every other key goes to Azure through `extra_body`. -const OPENAI_PARAMS: &[&str] = &[ - "functions", - "function_call", - "temperature", - "top_p", - "n", - "stream", - "stream_options", - "stop", - "max_tokens", - "max_completion_tokens", - "modalities", - "prediction", - "audio", - "presence_penalty", - "frequency_penalty", - "logit_bias", - "user", - "response_format", - "seed", - "tools", - "tool_choice", - "max_retries", - "logprobs", - "top_logprobs", - "extra_headers", - "parallel_tool_calls", - "reasoning_effort", - "verbosity", - "thinking", - "web_search_options", - "include_server_side_tool_invocations", - "service_tier", - "safety_identifier", - "prompt_cache_key", - "prompt_cache_retention", - "store", - "metadata", - "context_management", -]; - -/// `PROVIDER_UNVALIDATED_PARAMS`: never refused, only left out when unsupported. -const UNVALIDATED_PARAMS: &[&str] = &["user", "stream_options", "stream", "max_retries"]; - /// `AzureOpenAIConfig.get_supported_openai_params`. const AZURE_GPT_PARAMS: &[&str] = &[ "temperature", @@ -410,6 +369,7 @@ impl AzureChatRequest<'_> { let model = self.deployment; if model.starts_with(RESPONSES_ROUTE) || model_info::field(self.info(), &format!("azure/{model}"), "mode") + .as_ref() .and_then(Value::as_str) == Some("responses") { @@ -830,7 +790,9 @@ fn openai_gpt_params(model: &str, info: Option<&dyn ModelInfo>) -> Vec<&'static .split_once(RESPONSES_ROUTE) .map_or(model, |(_, rest)| rest); if matches!( - model_info::field(info, catalog_name, "litellm_provider").and_then(Value::as_str), + model_info::field(info, catalog_name, "litellm_provider") + .as_ref() + .and_then(Value::as_str), Some("openai" | "text-completion-openai") ) { params.push("user"); @@ -913,14 +875,6 @@ fn azure_messages(messages: Value) -> Value { Value::Array(messages) } -/// The request's OpenAI params in `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` order, the order -/// Python maps them in. -fn ordered(params: &Map) -> impl Iterator { - OPENAI_PARAMS - .iter() - .filter_map(|name| params.get(*name).map(|value| (*name, value))) -} - /// `_map_openai_params`: every supported param still unmapped goes as sent. fn copy_supported( non_default: &mut Map, @@ -936,40 +890,6 @@ fn copy_supported( } } -/// Python's truthiness for a JSON value. -fn truthy(value: &Value) -> bool { - match value { - Value::Null => false, - Value::Bool(flag) => *flag, - Value::Number(number) => number.as_f64() != Some(0.0), - Value::String(text) => !text.is_empty(), - Value::Array(items) => !items.is_empty(), - Value::Object(object) => !object.is_empty(), - } -} - -/// `normalize_drop_params`: a boolean, or a config's `"true"`. -fn request_drop_params(value: Option) -> bool { - match value { - Some(Value::Bool(flag)) => flag, - Some(Value::String(text)) => text.trim().eq_ignore_ascii_case("true"), - _ => false, - } -} - -fn string_list(value: Option) -> Vec { - match value { - Some(Value::Array(items)) => items - .into_iter() - .filter_map(|item| match item { - Value::String(text) => Some(text), - _ => None, - }) - .collect(), - _ => Vec::new(), - } -} - #[cfg(test)] mod tests { use std::collections::HashMap; @@ -997,8 +917,8 @@ mod tests { } impl ModelInfo for Entries { - fn entry(&self, model: &str) -> Option<&Map> { - self.0.get(model) + fn entry(&self, model: &str) -> Option>> { + self.0.get(model).map(std::borrow::Cow::Borrowed) } fn declares(&self, key: &str) -> bool { diff --git a/litellm-rust/crates/llms/src/base_llm/chat/media.rs b/litellm-rust/crates/llms/src/base_llm/chat/media.rs new file mode 100644 index 00000000000..8f631b5bb89 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/chat/media.rs @@ -0,0 +1,209 @@ +//! Media a chat message points at. Python downloads an image or file URL a provider cannot +//! take by reference while it builds the request; here the host downloads them first, from +//! the list a config names, and hands the messages back with each one inlined as a data URL. + +use std::collections::HashMap; + +use litellm_llms_types::formats::chat_completions::{ChatMessage, ChatMessageContent}; +use serde_json::Value; + +/// Python's `MAX_IMAGE_URL_DOWNLOAD_SIZE_MB`. +pub const MAX_MEDIA_DOWNLOAD_BYTES: u64 = 50 * 1024 * 1024; + +/// A part's `image_url`, given as a string or as `{"url": ...}`. +pub fn image_url(part: &Value) -> Option<&str> { + let image_url = part.get("image_url")?; + image_url + .as_str() + .or_else(|| image_url.get("url").and_then(Value::as_str)) +} + +/// `data:;base64,` split into its media type and payload, the way Python's +/// `split("data:")[1].split(";base64,")` reads it. +pub fn data_url(url: &str) -> Option<(&str, &str)> { + let (_, rest) = url.split_once("data:")?; + let (media_type, data) = rest.split_once(";base64,")?; + (!data.contains(";base64,")).then_some((media_type, data)) +} + +fn parts(message: &ChatMessage) -> &[Value] { + match &message.content { + Some(ChatMessageContent::Parts(parts)) => parts, + _ => &[], + } +} + +/// The URLs of `image_url` parts (and, with `files`, of `file.file_id` / `file.file_data`) +/// in the messages `roles` names that `remote` says must be downloaded, in order, each once. +pub fn remote_urls( + messages: &[ChatMessage], + roles: &[&str], + files: bool, + remote: impl Fn(&str) -> bool, +) -> Vec { + let mut urls: Vec = Vec::new(); + let mut add = |url: &str| { + if remote(url) && !urls.iter().any(|seen| seen == url) { + urls.push(url.to_owned()); + } + }; + for message in messages + .iter() + .filter(|message| roles.contains(&message.role.as_str())) + { + for part in parts(message) { + match part.get("type").and_then(Value::as_str) { + Some("image_url") => { + if let Some(url) = image_url(part) { + add(url); + } + } + Some("file") if files => { + let file = part.get("file"); + let source = ["file_id", "file_data"].into_iter().find_map(|key| { + file.and_then(|file| file.get(key)).and_then(Value::as_str) + }); + if let Some(url) = source { + add(url); + } + } + _ => {} + } + } + } + urls +} + +/// Every reference to a downloaded URL replaced by its data URL. +pub fn inline_media(messages: &mut [ChatMessage], downloaded: &HashMap) { + if downloaded.is_empty() { + return; + } + for message in messages { + let Some(ChatMessageContent::Parts(parts)) = &mut message.content else { + continue; + }; + for part in parts { + match part.get("type").and_then(Value::as_str) { + Some("image_url") => match part.get_mut("image_url") { + Some(Value::String(url)) => { + if let Some(data) = downloaded.get(url.as_str()) { + *url = data.clone(); + } + } + Some(Value::Object(image)) => { + if let Some(Value::String(url)) = image.get_mut("url") + && let Some(data) = downloaded.get(url.as_str()) + { + *url = data.clone(); + } + } + _ => {} + }, + Some("file") => { + if let Some(Value::Object(file)) = part.get_mut("file") { + for key in ["file_id", "file_data"] { + if let Some(Value::String(url)) = file.get_mut(key) + && let Some(data) = downloaded.get(url.as_str()) + { + *url = data.clone(); + } + } + } + } + _ => {} + } + } + } +} + +/// The media type a download is inlined with: the response's own, or one guessed from the +/// URL's extension the way Python's `infer_content_type_from_url_and_content` falls back. +pub fn downloaded_media_type(url: &str, content_type: &str) -> String { + let declared = content_type + .split(';') + .next() + .unwrap_or_default() + .trim() + .to_ascii_lowercase(); + if !declared.is_empty() && declared != "application/octet-stream" && declared.contains('/') { + return declared; + } + let path = url + .split(['?', '#']) + .next() + .unwrap_or(url) + .to_ascii_lowercase(); + let extension = path.rsplit('.').next().unwrap_or_default(); + match extension { + "png" => "image/png", + "jpg" | "jpeg" => "image/jpeg", + "gif" => "image/gif", + "webp" => "image/webp", + "pdf" => "application/pdf", + "txt" => "text/plain", + "csv" => "text/csv", + "html" | "htm" => "text/html", + "md" => "text/markdown", + "mp4" => "video/mp4", + _ => "image/jpeg", + } + .to_owned() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn messages(value: Value) -> Vec { + serde_json::from_value(value).unwrap() + } + + #[rstest] + fn remote_urls_are_listed_once_and_inlined() { + let mut list = messages(json!([ + {"role": "user", "content": [ + {"type": "image_url", "image_url": "http://a/x.png"}, + {"type": "image_url", "image_url": {"url": "http://a/x.png"}}, + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AA=="}}, + {"type": "file", "file": {"file_id": "https://b/doc.pdf"}} + ]}, + {"role": "assistant", "content": [{"type": "image_url", "image_url": "http://c"}]} + ])); + let urls = remote_urls(&list, &["user", "tool"], true, |url| { + url.starts_with("http") + }); + assert_eq!(urls, vec!["http://a/x.png", "https://b/doc.pdf"]); + let downloaded = HashMap::from([( + "http://a/x.png".to_owned(), + "data:image/png;base64,Zm8=".to_owned(), + )]); + inline_media(&mut list, &downloaded); + let content = serde_json::to_value(&list[0].content).unwrap(); + assert_eq!(content[0]["image_url"], "data:image/png;base64,Zm8="); + assert_eq!(content[1]["image_url"]["url"], "data:image/png;base64,Zm8="); + assert_eq!(content[3]["file"]["file_id"], "https://b/doc.pdf"); + } + + #[rstest] + #[case("data:image/png;base64,AAAA", Some(("image/png", "AAAA")))] + #[case("https://x", None)] + fn data_urls_split(#[case] url: &str, #[case] expected: Option<(&str, &str)>) { + assert_eq!(data_url(url), expected); + } + + #[rstest] + #[case("https://x/a.PNG?x=1", "", "image/png")] + #[case("https://x/a", "image/webp; charset=x", "image/webp")] + #[case("https://x/a.pdf", "application/octet-stream", "application/pdf")] + fn media_types_fall_back_to_the_extension( + #[case] url: &str, + #[case] content_type: &str, + #[case] expected: &str, + ) { + assert_eq!(downloaded_media_type(url, content_type), expected); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/chat/mod.rs b/litellm-rust/crates/llms/src/base_llm/chat/mod.rs index fa7df180f50..4c43dde0757 100644 --- a/litellm-rust/crates/llms/src/base_llm/chat/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/chat/mod.rs @@ -1,2 +1,5 @@ +pub mod media; +pub mod params; +pub mod response_format; pub mod streaming; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/base_llm/chat/params.rs b/litellm-rust/crates/llms/src/base_llm/chat/params.rs new file mode 100644 index 00000000000..9ffacff1611 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/chat/params.rs @@ -0,0 +1,434 @@ +//! Python's `get_optional_params` for the providers chat translates: the request's keys are +//! sorted into OpenAI params, keys LiteLLM consumes itself, and provider-specific keys that +//! pass through; an OpenAI param the provider does not take is dropped under `drop_params` +//! and refused otherwise. + +use serde_json::{Map, Value}; + +use crate::{ + Error, ErrorDetail, base_llm::model_info::ModelInfo, + openai::chat::transformation::is_litellm_owned, +}; + +/// The keys of `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` a request carries as OpenAI params, in +/// that table's order, which is the order Python maps them in. +pub const OPENAI_PARAMS: &[&str] = &[ + "functions", + "function_call", + "temperature", + "top_p", + "n", + "stream", + "stream_options", + "stop", + "max_tokens", + "max_completion_tokens", + "modalities", + "prediction", + "audio", + "presence_penalty", + "frequency_penalty", + "logit_bias", + "user", + "response_format", + "seed", + "tools", + "tool_choice", + "max_retries", + "logprobs", + "top_logprobs", + "extra_headers", + "parallel_tool_calls", + "reasoning_effort", + "verbosity", + "thinking", + "web_search_options", + "include_server_side_tool_invocations", + "service_tier", + "safety_identifier", + "prompt_cache_key", + "prompt_cache_retention", + "store", + "metadata", + "context_management", +]; + +/// `PROVIDER_UNVALIDATED_PARAMS`: never refused, only left out when unsupported. +pub const UNVALIDATED_PARAMS: &[&str] = &["user", "stream_options", "stream", "max_retries"]; + +/// The tool Python adds when `response_format` asks for JSON and the model has no native +/// structured output. +pub const RESPONSE_FORMAT_TOOL_NAME: &str = "json_tool_call"; + +/// Python's `DEFAULT_MAX_TOKENS`, added to a thinking budget when the caller set no limit. +pub const DEFAULT_MAX_TOKENS: u64 = 4096; + +/// The request's OpenAI params in `DEFAULT_CHAT_COMPLETION_PARAM_VALUES` order. +pub fn ordered(params: &Map) -> impl Iterator { + OPENAI_PARAMS + .iter() + .filter_map(|name| params.get(*name).map(|value| (*name, value))) +} + +/// Python's truthiness for a JSON value. +pub fn truthy(value: &Value) -> bool { + match value { + Value::Null => false, + Value::Bool(flag) => *flag, + Value::Number(number) => number.as_f64() != Some(0.0), + Value::String(text) => !text.is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(object) => !object.is_empty(), + } +} + +/// `normalize_drop_params`: a boolean, or a config's `"true"`. +pub fn request_drop_params(value: Option) -> bool { + match value { + Some(Value::Bool(flag)) => flag, + Some(Value::String(text)) => text.trim().eq_ignore_ascii_case("true"), + _ => false, + } +} + +pub fn string_list(value: Option) -> Vec { + match value { + Some(Value::Array(items)) => items + .into_iter() + .filter_map(|item| match item { + Value::String(text) => Some(text), + _ => None, + }) + .collect(), + _ => Vec::new(), + } +} + +/// What the host tells a config about the request beyond its params. +#[derive(Clone, Copy, Default)] +pub struct ChatRequestContext<'a> { + /// `drop_params` from the proxy settings or the deployment. `None` when the host cannot + /// say, as when Python hosts the call: a request Python would drop or refuse a param of + /// then goes back to Python instead. + pub drop_params: Option, + /// The model cost map, which several param rules read. + pub model_info: Option<&'a dyn ModelInfo>, +} + +/// Whether a param the provider does not take is dropped, refused, or left to Python. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct DropPolicy(Option); + +impl DropPolicy { + pub fn new(drop_params: Option) -> Self { + Self(drop_params) + } + + /// Whether unsupported params are dropped. + pub fn drops(self) -> bool { + self.0 == Some(true) + } + + /// Whether the host said what `drop_params` is. + pub fn known(self) -> bool { + self.0.is_some() + } + + /// A param Python drops under `drop_params` and refuses otherwise: `Ok` means drop it. + pub fn unsupported(self, refusal: impl FnOnce() -> Error) -> Result<(), Error> { + match self.0 { + Some(true) => Ok(()), + Some(false) => Err(refusal()), + None => Err(Error::Unsupported( + "a param Python drops or refuses by drop_params", + )), + } + } +} + +/// Python's `UnsupportedParamsError` for one param with its own message. +pub fn unsupported_value(message: impl Into) -> Error { + Error::InvalidRequest(ErrorDetail::Message(message.into())) +} + +/// A request's params as `get_optional_params` sorts them. +#[derive(Debug, Default)] +pub struct SortedParams { + /// The OpenAI params, refused or dropped ones removed. + pub openai: Map, + /// Every other key, sent to the provider as given. + pub provider: Map, + /// Keys the config consumes as call configuration. + pub config: Map, + /// `allowed_openai_params`: sent as given even when the provider does not list them. + pub allowed: Vec, + pub drop: DropPolicy, +} + +/// Sorts `params` and refuses or drops the OpenAI params `supported` leaves out. +pub fn sort_params( + params: Map, + config_params: &[&str], + context: &ChatRequestContext<'_>, + supported: impl FnOnce(&Map) -> Result, Error>, + provider: &'static str, + model: &str, +) -> Result { + let mut params = params; + let request_drop = request_drop_params(params.remove("drop_params")); + let drop = DropPolicy::new(match context.drop_params { + Some(configured) => Some(configured || request_drop), + None if request_drop => Some(true), + None => None, + }); + let allowed = string_list(params.remove("allowed_openai_params")); + let additional_drop = string_list(params.remove("additional_drop_params")); + let mut sorted = SortedParams { + allowed, + drop, + ..SortedParams::default() + }; + for (name, value) in params { + if config_params.contains(&name.as_str()) { + sorted.config.insert(name, value); + continue; + } + if value.is_null() || is_litellm_owned(&name) || additional_drop.contains(&name) { + continue; + } + match OPENAI_PARAMS.contains(&name.as_str()) { + true => sorted.openai.insert(name, value), + false => sorted.provider.insert(name, value), + }; + } + let supported = supported(&sorted.openai)?; + let unsupported: Vec = sorted + .openai + .iter() + .filter(|(name, _)| { + !supported.contains(&name.as_str()) && !sorted.allowed.iter().any(|a| a == *name) + }) + .map(|(name, _)| name.clone()) + .collect(); + let refused: Vec = unsupported + .iter() + .filter(|name| { + !UNVALIDATED_PARAMS.contains(&name.as_str()) + && !(name.as_str() == "n" + && sorted.openai.get(name.as_str()).and_then(Value::as_f64) == Some(1.0)) + }) + .cloned() + .collect(); + if !refused.is_empty() { + drop.unsupported(|| { + Error::InvalidRequest(ErrorDetail::UnsupportedParams { + provider, + params: refused.clone(), + model: model.to_string(), + }) + })?; + } + for name in unsupported { + sorted.openai.remove(&name); + } + Ok(sorted) +} + +/// A capability flag the cost map sets for the first of `candidates` it knows. +/// +/// Without a cost map Python's own lookups have nothing to go on either, so the answer is +/// `None`, except when the host could still fall back to Python, which does have one. +pub fn capability( + context: &ChatRequestContext<'_>, + candidates: &[String], + key: &str, +) -> Result, Error> { + let Some(info) = context.model_info else { + return match context.drop_params { + None => Err(Error::Unsupported( + "a request whose translation reads the model cost map", + )), + Some(_) => Ok(None), + }; + }; + Ok(candidates + .iter() + .filter_map(|candidate| info.entry(candidate)) + .find_map(|entry| entry.get(key).and_then(Value::as_bool))) +} + +/// A numeric field of the first of `candidates` the cost map knows. +pub fn model_field( + context: &ChatRequestContext<'_>, + candidates: &[String], + key: &str, +) -> Option { + let info = context.model_info?; + candidates + .iter() + .filter_map(|candidate| info.entry(candidate)) + .find_map(|entry| entry.get(key).filter(|value| !value.is_null()).cloned()) +} + +/// Whether the cost map has an entry for any of `candidates`. +pub fn model_known(context: &ChatRequestContext<'_>, candidates: &[String]) -> bool { + context + .model_info + .is_some_and(|info| candidates.iter().any(|c| info.entry(c).is_some())) +} + +/// `json.dumps(value)` with Python's default separators and ASCII escaping. +pub fn python_dumps(value: &Value) -> String { + litellm_python_compat::json::dumps(&litellm_python_compat::json::from_json(value.clone())) + .unwrap_or_else(|_| value.to_string()) +} + +/// `json.loads` of tool call arguments, as `parse_tool_call_arguments` reads them: empty +/// arguments are `{}`. +pub fn parse_tool_arguments(arguments: Option<&Value>) -> Result { + match arguments { + None | Some(Value::Null) => Ok(Value::Object(Map::new())), + Some(Value::String(text)) if text.trim().is_empty() => Ok(Value::Object(Map::new())), + Some(Value::String(text)) => serde_json::from_str(text).map_err(|error| { + Error::InvalidRequest(ErrorDetail::Message(format!( + "tool call arguments are not valid JSON: {error}" + ))) + }), + Some(other) => Ok(other.clone()), + } +} + +/// `is_thinking_enabled` over a params map. +pub fn is_thinking_enabled(params: &Map) -> bool { + let thinking = params.get("thinking"); + thinking == Some(&Value::Bool(true)) + || thinking + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("enabled") + || params + .get("reasoning_effort") + .is_some_and(|effort| !effort.is_null()) +} + +/// The JSON schema a `response_format` names: `response_schema`, or `json_schema.schema`. +pub fn response_format_schema(value: &Value) -> Option { + if let Some(schema) = value.get("response_schema") { + return Some(schema.clone()); + } + value + .get("json_schema") + .and_then(|json_schema| json_schema.get("schema")) + .cloned() +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn object(value: Value) -> Map { + match value { + Value::Object(map) => map, + other => panic!("expected an object, got {other}"), + } + } + + fn sort(params: Value, drop_params: Option) -> Result { + sort_params( + object(params), + &["aws_region_name"], + &ChatRequestContext { + drop_params, + model_info: None, + }, + |_| Ok(vec!["temperature", "max_tokens"]), + "anthropic", + "claude", + ) + } + + #[rstest] + fn params_sort_into_openai_provider_config_and_owned() { + let sorted = sort( + json!({"temperature": 0.5, "top_k": 3, "aws_region_name": "us-east-1", + "metadata": {"x": 1}, "mock_response": "hi", "_litellm_x": 1, "user": "u", "seed": null}), + Some(false), + ) + .unwrap(); + assert_eq!(sorted.openai, object(json!({"temperature": 0.5}))); + assert_eq!(sorted.provider, object(json!({"top_k": 3}))); + assert_eq!( + sorted.config, + object(json!({"aws_region_name": "us-east-1"})) + ); + } + + #[rstest] + #[case::refused(Some(false), false)] + #[case::dropped(Some(true), true)] + #[case::request_drop(None, true)] + fn an_unsupported_param_is_refused_or_dropped( + #[case] drop_params: Option, + #[case] dropped: bool, + ) { + let mut body = json!({"temperature": 0.5, "seed": 3}); + if drop_params.is_none() { + body["drop_params"] = json!(true); + } + let result = sort(body, drop_params); + match dropped { + true => assert_eq!(result.unwrap().openai, object(json!({"temperature": 0.5}))), + false => assert!(matches!( + result, + Err(Error::InvalidRequest(ErrorDetail::UnsupportedParams { ref params, .. })) + if params == &vec!["seed".to_string()] + )), + } + } + + #[rstest] + fn an_unknown_drop_policy_falls_back() { + assert!(matches!( + sort(json!({"seed": 3}), None), + Err(Error::Unsupported(_)) + )); + } + + #[rstest] + fn allowed_and_additional_drop_params_are_honoured() { + let sorted = sort( + json!({"seed": 3, "top_k": 2, "allowed_openai_params": ["seed"], "additional_drop_params": ["top_k"]}), + Some(false), + ) + .unwrap(); + assert_eq!(sorted.openai, object(json!({"seed": 3}))); + assert!(sorted.provider.is_empty()); + assert_eq!(sorted.allowed, vec!["seed".to_string()]); + } + + #[rstest] + #[case::n_of_one(json!({"n": 1}), true)] + #[case::n_of_two(json!({"n": 2}), false)] + #[case::user(json!({"user": "u"}), true)] + fn unvalidated_params_and_one_choice_are_never_refused( + #[case] body: Value, + #[case] accepted: bool, + ) { + let result = sort(body, Some(false)); + assert_eq!(result.is_ok(), accepted, "{result:?}"); + if let Ok(sorted) = result { + assert!(sorted.openai.is_empty()); + } + } + + #[rstest] + fn python_dumps_matches_json_dumps() { + assert_eq!( + python_dumps(&json!({"b": "é", "a": [1, 2.5]})), + r#"{"b": "\u00e9", "a": [1, 2.5]}"# + ); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/chat/response_format.rs b/litellm-rust/crates/llms/src/base_llm/chat/response_format.rs new file mode 100644 index 00000000000..a7f139c123a --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/chat/response_format.rs @@ -0,0 +1,445 @@ +//! `response_format` answers delivered through the `json_tool_call` tool, as Python's +//! `base_utils.py` reads them back: the schema's shape, the envelopes a model wraps the +//! answer in, and an answer split across several tool calls. + +use std::collections::BTreeSet; + +use serde_json::{Map, Value}; + +use crate::base_llm::chat::params::python_dumps; + +/// `ResponseFormatSchema`: what the requested schema declares, enough to tell an answer +/// from an envelope around it. +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct ResponseFormatSchema { + pub property_names: BTreeSet, + pub required_names: BTreeSet, + pub allows_undeclared_keys: bool, +} + +impl ResponseFormatSchema { + /// `response_format_schema`: `None` unless `value` is an object whose `properties`, if + /// present, is one too. + pub fn from_schema(value: &Value) -> Option { + let document = value.as_object()?; + let properties = match document.get("properties") { + None => Map::new(), + Some(Value::Object(properties)) => properties.clone(), + Some(_) => return None, + }; + let required_names = match document.get("required") { + Some(Value::Array(items)) if items.iter().all(Value::is_string) => items + .iter() + .filter_map(Value::as_str) + .map(str::to_owned) + .collect(), + _ => BTreeSet::new(), + }; + let additional = document.get("additionalProperties"); + let allows_undeclared_keys = additional == Some(&Value::Bool(true)) + || additional.is_some_and(Value::is_object) + || document + .get("patternProperties") + .and_then(Value::as_object) + .is_some_and(|patterns| !patterns.is_empty()); + Some(Self { + property_names: properties.keys().cloned().collect(), + required_names, + allows_undeclared_keys, + }) + } + + fn accepts_a_lone_undeclared_key(&self) -> bool { + self.allows_undeclared_keys && self.required_names.is_empty() + } +} + +const ENVELOPE_NAME_KEYS: [&str; 2] = ["name", "tool_name"]; + +/// `_json_object_or_encoded_object`. +fn object_or_encoded_object(value: &Value) -> Option> { + match value { + Value::String(text) => match serde_json::from_str(text) { + Ok(Value::Object(object)) => Some(object), + _ => None, + }, + Value::Object(object) => Some(object.clone()), + _ => None, + } +} + +struct EnvelopeLayer { + inner: Map, + names_the_tool: bool, +} + +fn envelope_layer( + payload: &Map, + schema: &ResponseFormatSchema, + tool_name: &str, +) -> Option { + let name_keys: BTreeSet<&str> = ENVELOPE_NAME_KEYS + .into_iter() + .filter(|key| payload.get(*key).and_then(Value::as_str) == Some(tool_name)) + .collect(); + let label_keys: BTreeSet<&str> = match (name_keys.is_empty(), schema.required_names.is_empty()) + { + (false, _) => name_keys.clone(), + (true, true) => BTreeSet::new(), + (true, false) => payload + .iter() + .filter(|(key, value)| { + value.is_string() + && !schema.property_names.contains(*key) + && !schema.required_names.contains(*key) + }) + .map(|(key, _)| key.as_str()) + .collect(), + }; + let wrapper_keys: Vec<&String> = payload + .keys() + .filter(|key| !label_keys.contains(key.as_str())) + .collect(); + let [wrapper] = wrapper_keys.as_slice() else { + return None; + }; + if schema.property_names.contains(*wrapper) { + return None; + } + let wrapped = &payload[wrapper.as_str()]; + let inner = match name_keys.is_empty() { + false => object_or_encoded_object(wrapped), + true => wrapped.as_object().cloned(), + }?; + Some(EnvelopeLayer { + inner, + names_the_tool: !name_keys.is_empty(), + }) +} + +fn holds_the_schema_object(layer: &EnvelopeLayer, schema: &ResponseFormatSchema) -> bool { + layer.names_the_tool + || (!schema.accepts_a_lone_undeclared_key() + && layer + .inner + .keys() + .any(|key| schema.property_names.contains(key))) +} + +fn schema_object_inside_envelope( + payload: &Map, + schema: &ResponseFormatSchema, + tool_name: &str, +) -> Option> { + let mut layer = envelope_layer(payload, schema, tool_name); + let mut first = true; + while let Some(current) = layer { + if !first && schema.property_names.is_empty() { + return None; + } + first = false; + if holds_the_schema_object(¤t, schema) { + return Some(current.inner); + } + layer = envelope_layer(¤t.inner, schema, tool_name); + } + None +} + +/// `response_format_envelope_contents`. +pub fn envelope_contents( + payload: &Value, + schema: &ResponseFormatSchema, + tool_name: &str, +) -> Option { + let object = payload.as_object()?; + if let Some(inner) = schema_object_inside_envelope(object, schema, tool_name) { + return Some(Value::Object(inner)); + } + if schema.property_names.contains("values") { + return None; + } + object + .get("values") + .filter(|values| !values.is_null()) + .cloned() +} + +/// `unwrap_response_format_envelope`. +pub fn unwrap_envelope(payload: Value, schema: &ResponseFormatSchema, tool_name: &str) -> Value { + envelope_contents(&payload, schema, tool_name).unwrap_or(payload) +} + +fn field_names( + blocks: &[Map], + key: &str, + schema: &ResponseFormatSchema, +) -> Option> { + let names: Vec = blocks + .iter() + .filter_map(|block| block[key].as_str()) + .filter(|name| schema.property_names.contains(*name)) + .map(str::to_owned) + .collect(); + let distinct: BTreeSet<&String> = names.iter().collect(); + (distinct.len() == blocks.len()).then_some(names) +} + +fn is_a_label_every_block_repeats(blocks: &[Map], key: &str) -> bool { + blocks[0][key].is_string() && blocks.iter().all(|block| block[key] == blocks[0][key]) +} + +fn value_key(blocks: &[Map], keys: &[&String]) -> Option { + if let [only] = keys { + return Some((*only).clone()); + } + let varying: Vec<&&String> = keys + .iter() + .filter(|key| !is_a_label_every_block_repeats(blocks, key)) + .collect(); + match varying.as_slice() { + [only] => Some((**only).clone()), + _ => None, + } +} + +fn split_answer_fields( + blocks: &[Map], + schema: &ResponseFormatSchema, +) -> Option> { + let keys: BTreeSet<&String> = blocks[0].keys().collect(); + if keys.iter().any(|key| schema.property_names.contains(*key)) + || blocks + .iter() + .any(|block| block.keys().collect::>() != keys) + { + return None; + } + // Python iterates a frozenset here; the outcome only depends on there being one match. + let named: Vec<(&String, Vec)> = keys + .iter() + .filter_map(|key| Some((*key, field_names(blocks, key, schema)?))) + .collect(); + let [(name_key, names)] = named.as_slice() else { + return None; + }; + let rest: Vec<&String> = keys.iter().copied().filter(|key| key != name_key).collect(); + let value_key = value_key(blocks, &rest)?; + Some( + names + .iter() + .cloned() + .zip(blocks.iter().map(|block| block[value_key.as_str()].clone())) + .collect(), + ) +} + +fn merged_partial_answers( + blocks: &[Map], + schema: &ResponseFormatSchema, +) -> Option> { + let keys: BTreeSet<&String> = blocks.iter().flat_map(Map::keys).collect(); + if !keys.iter().all(|key| schema.property_names.contains(*key)) + || keys.len() != blocks.iter().map(Map::len).sum::() + { + return None; + } + Some( + blocks + .iter() + .flat_map(|block| block.iter().map(|(k, v)| (k.clone(), v.clone()))) + .collect(), + ) +} + +/// `split_response_format_answer`: one answer spread over several tool calls. +pub fn split_answer( + encoded_blocks: &[String], + schema: &ResponseFormatSchema, +) -> Option> { + let blocks: Vec> = encoded_blocks + .iter() + .map(|encoded| object_or_encoded_object(&Value::String(encoded.clone()))) + .collect::>()?; + if blocks.len() < 2 { + return None; + } + split_answer_fields(&blocks, schema).or_else(|| merged_partial_answers(&blocks, schema)) +} + +/// Anthropic's `_convert_tool_response_to_message`: the message content the `json_tool_call` +/// calls' arguments become. +pub fn json_tool_answer(arguments: &[String], schema: Option<&ResponseFormatSchema>) -> String { + let first = arguments.first().cloned().unwrap_or_default(); + if let Some(split) = schema.and_then(|schema| split_answer(arguments, schema)) { + return python_dumps(&Value::Object(split)); + } + let Ok(payload) = serde_json::from_str::(&first) else { + return first; + }; + match schema { + Some(schema) => python_dumps(&unwrap_envelope( + payload, + schema, + crate::base_llm::chat::params::RESPONSE_FORMAT_TOOL_NAME, + )), + None => { + let values = payload + .as_object() + .and_then(|object| object.get("values")) + .filter(|values| !values.is_null()) + .cloned(); + python_dumps(values.as_ref().unwrap_or(&payload)) + } + } +} + +/// `unpack_defs`: every `$ref` into `$defs` / `definitions` replaced by a copy of its +/// target, nested definitions shadowing outer ones, a circular reference left as it is. +pub fn unpack_defs(schema: &mut Value, defs: &Map) { + fn scoped(node: &Map, outer: &Map) -> Map { + let mut defs = outer.clone(); + for key in ["$defs", "definitions"] { + if let Some(Value::Object(local)) = node.get(key) { + defs.extend(local.iter().map(|(k, v)| (k.clone(), v.clone()))); + } + } + defs + } + fn walk(node: &mut Value, defs: &Map, chain: &BTreeSet) { + match node { + Value::Object(object) => { + if let Some(reference) = object.get("$ref").and_then(Value::as_str) { + let name = reference.rsplit('/').next().unwrap_or(reference).to_owned(); + if chain.contains(&name) { + return; + } + let Some(target) = defs.get(&name).cloned() else { + return; + }; + let child_defs = match &target { + Value::Object(target) => scoped(target, defs), + _ => defs.clone(), + }; + *node = target; + let mut chain = chain.clone(); + chain.insert(name); + walk(node, &child_defs, &chain); + return; + } + if object.contains_key("$ref") { + return; + } + let current = scoped(object, defs); + for value in object.values_mut() { + walk(value, ¤t, chain); + } + } + Value::Array(items) => { + for item in items { + walk(item, defs, chain); + } + } + _ => {} + } + } + let root = match schema { + Value::Object(object) => scoped(object, defs), + _ => defs.clone(), + }; + walk(schema, &root, &BTreeSet::new()); +} + +/// `unpack_legacy_defs`: refs into `definitions` or OpenAPI `components.schemas` inlined, +/// those blocks removed. `$defs` is left alone. +pub fn unpack_legacy_defs(schema: &mut Map) { + let has_components = schema + .get("components") + .and_then(|components| components.get("schemas")) + .is_some(); + if !schema.contains_key("definitions") && !has_components { + return; + } + let mut defs = match schema.remove("components") { + Some(Value::Object(mut components)) => match components.remove("schemas") { + Some(Value::Object(schemas)) => schemas, + _ => Map::new(), + }, + _ => Map::new(), + }; + if let Some(Value::Object(definitions)) = schema.remove("definitions") { + defs.extend(definitions); + } + let mut value = Value::Object(std::mem::take(schema)); + unpack_defs(&mut value, &defs); + if let Value::Object(object) = value { + *schema = object; + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn schema(value: Value) -> ResponseFormatSchema { + ResponseFormatSchema::from_schema(&value).unwrap() + } + + #[rstest] + #[case::bare(json!({"name": "a"}), json!({"name": "a"}))] + #[case::named_envelope(json!({"name": "json_tool_call", "arguments": {"name": "a"}}), json!({"name": "a"}))] + #[case::encoded_envelope(json!({"tool_name": "json_tool_call", "parameters": "{\"name\": \"a\"}"}), json!({"name": "a"}))] + #[case::wrapper(json!({"answer": {"name": "a"}}), json!({"name": "a"}))] + #[case::values(json!({"values": {"x": 1}}), json!({"x": 1}))] + fn envelopes_are_unwrapped(#[case] payload: Value, #[case] expected: Value) { + let schema = schema( + json!({"type": "object", "properties": {"name": {"type": "string"}}, "required": ["name"]}), + ); + assert_eq!( + unwrap_envelope(payload, &schema, "json_tool_call"), + expected + ); + } + + #[rstest] + fn an_answer_split_over_named_fields_is_joined() { + let schema = schema(json!({"properties": {"a": {}, "b": {}}})); + let blocks = vec![ + r#"{"field": "a", "value": 1}"#.to_owned(), + r#"{"field": "b", "value": 2}"#.to_owned(), + ]; + assert_eq!( + split_answer(&blocks, &schema), + Some(json!({"a": 1, "b": 2}).as_object().unwrap().clone()) + ); + let partial = vec![r#"{"a": 1}"#.to_owned(), r#"{"b": 2}"#.to_owned()]; + assert_eq!( + json_tool_answer(&partial, Some(&schema)), + r#"{"a": 1, "b": 2}"# + ); + } + + #[rstest] + fn an_undecodable_answer_is_returned_as_sent() { + assert_eq!(json_tool_answer(&["{bad".to_owned()], None), "{bad"); + assert_eq!( + json_tool_answer(&[r#"{"values": [1]}"#.to_owned()], None), + "[1]" + ); + } + + #[rstest] + fn refs_are_inlined_and_cycles_left_alone() { + let mut value = + json!({"properties": {"a": {"$ref": "#/$defs/A"}, "n": {"$ref": "#/$defs/Node"}}}); + let defs = json!({"A": {"type": "string"}, "Node": {"properties": {"next": {"$ref": "#/$defs/Node"}}}}); + unpack_defs(&mut value, defs.as_object().unwrap()); + assert_eq!( + value, + json!({"properties": {"a": {"type": "string"}, "n": {"properties": {"next": {"$ref": "#/$defs/Node"}}}}}) + ); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/chat/streaming.rs b/litellm-rust/crates/llms/src/base_llm/chat/streaming.rs index 2faa5699751..1c938b33a1e 100644 --- a/litellm-rust/crates/llms/src/base_llm/chat/streaming.rs +++ b/litellm-rust/crates/llms/src/base_llm/chat/streaming.rs @@ -1,5 +1,7 @@ use std::collections::HashMap; +use crate::base_llm::chat::response_format::ResponseFormatSchema; + use bytes::Bytes; use futures_util::{StreamExt, stream::BoxStream}; use litellm_llms_types::formats::chat_completions::ChatCompletionChunk; @@ -35,6 +37,12 @@ pub struct StreamShape { pub json_mode: bool, pub speed: Option, pub tool_name_reverse_map: HashMap, + /// The schema of the `json_tool_call` tool a `response_format` added, which tells its + /// answer apart from an envelope around it. + pub response_format_schema: Option, + /// The text of a trailing assistant message sent with `prefix: true`, which Anthropic + /// continues and Python puts back in front of the answer. + pub prefix_prompt: Option, } /// A wire decoder paired with the iterator that turns its events into chat chunks. diff --git a/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs index e70001bf0be..77c89e0e85a 100644 --- a/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/base_llm/chat/transformation.rs @@ -5,7 +5,10 @@ use serde_json::{Map, Value}; use crate::{ Error, - base_llm::chat::streaming::{ChatStream, StreamShape}, + base_llm::chat::{ + params::ChatRequestContext, + streaming::{ChatStream, StreamShape}, + }, }; /// The provider-shaped request body a config produces. Named rather than a bare @@ -14,6 +17,8 @@ use crate::{ pub struct ProviderChatRequestData { pub body: Value, pub stream_shape: StreamShape, + /// Headers the request needs beyond the config's defaults, such as `anthropic-beta`. + pub headers: Vec<(String, String)>, } /// The raw provider response body handed back to a config for normalization. @@ -42,8 +47,11 @@ pub struct Unsupported(pub &'static str); pub trait BaseConfig: Sync { fn secret_names(&self) -> Vec<&'static str>; - /// Supported OpenAI parameter names paired with their provider names. - fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)]; + /// Supported OpenAI parameter names paired with their provider names, for a config + /// whose requests the host checks against an allowlist. + fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)] { + &[] + } fn get_complete_url( &self, @@ -58,14 +66,23 @@ pub trait BaseConfig: Sync { model: &str, messages: Vec, optional_params: Map, + context: &ChatRequestContext<'_>, ) -> Result; + /// `shape` is what the request decided about its answer, the same one a stream reads. fn transform_response( &self, model: &str, response: ProviderChatResponseData, + shape: &StreamShape, ) -> Result; + /// The media URLs the messages point at that Python downloads while building this + /// provider's request. The host fetches them and inlines each as a data URL first. + fn remote_media(&self, _messages: &[ChatMessage]) -> Vec { + Vec::new() + } + /// `None` means this config has no streaming path yet, so the host keeps the request. fn model_response_iterator(&self, _shape: StreamShape) -> Option { None diff --git a/litellm-rust/crates/llms/src/base_llm/model_info.rs b/litellm-rust/crates/llms/src/base_llm/model_info.rs index a2861d138c7..3260c270306 100644 --- a/litellm-rust/crates/llms/src/base_llm/model_info.rs +++ b/litellm-rust/crates/llms/src/base_llm/model_info.rs @@ -1,11 +1,14 @@ //! What the model cost map says about a model, for configs whose request rules depend on it //! the way Python's `supports_*` helpers read `litellm.model_cost`. +use std::borrow::Cow; + use serde_json::{Map, Value}; pub trait ModelInfo: Send + Sync { - /// The cost map entry `model` names, if there is one. - fn entry(&self, model: &str) -> Option<&Map>; + /// The cost map entry `model` names, if there is one. A map held in memory lends it; one + /// read from elsewhere, such as Python's `litellm.model_cost`, hands over a copy. + fn entry(&self, model: &str) -> Option>>; /// Whether any entry of the map sets `key`. fn declares(&self, key: &str) -> bool; @@ -14,21 +17,22 @@ pub trait ModelInfo: Send + Sync { /// Python's `_supports_factory`: the entry sets `key` to `true`. An unknown model supports /// nothing. pub fn supports(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> bool { - field(info, model, key).and_then(Value::as_bool) == Some(true) + field(info, model, key).and_then(|value| value.as_bool()) == Some(true) } /// Python's `is_explicitly_disabled_factory`: the entry sets `key` to `false`. An unknown /// model or an unset key is not disabled. pub fn explicitly_disabled(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> bool { - field(info, model, key).and_then(Value::as_bool) == Some(false) + field(info, model, key).and_then(|value| value.as_bool()) == Some(false) } /// The value the entry for `model` sets for `key`. -pub fn field<'a>(info: Option<&'a dyn ModelInfo>, model: &str, key: &str) -> Option<&'a Value> { +pub fn field(info: Option<&dyn ModelInfo>, model: &str, key: &str) -> Option { info? .entry(model)? .get(key) .filter(|value| !value.is_null()) + .cloned() } /// Whether the map has an entry for `model`. diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_messages.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_messages.rs new file mode 100644 index 00000000000..67cbd088a7a --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_messages.rs @@ -0,0 +1,1343 @@ +//! Chat messages and tools as Converse takes them: Python's `_transform_system_message`, +//! `_neutralize_orphaned_tool_blocks`, `_bedrock_converse_messages_pt` and `_bedrock_tools_pt`. + +use std::collections::HashSet; + +use serde_json::{Map, Value, json}; +use sha2::{Digest, Sha256}; + +use crate::{ + Error, ErrorDetail, + anthropic::chat::messages::{CONVERTED_SYSTEM_NOTE, native_redacted_thinking_data}, + base_llm::chat::{params::truthy, response_format::unpack_defs}, + bedrock::chat::converse_params::{BedrockModel, make_valid_bedrock_tool_name}, +}; + +const SUPPORTED_IMAGE_FORMATS: [&str; 4] = ["png", "jpeg", "gif", "webp"]; +const SUPPORTED_DOCUMENT_FORMATS: [&str; 9] = [ + "pdf", "csv", "doc", "docx", "xls", "xlsx", "html", "txt", "md", +]; +const SUPPORTED_VIDEO_FORMATS: [&str; 9] = [ + "mp4", "mov", "mkv", "webm", "flv", "mpeg", "mpg", "wmv", "3gp", +]; +const DOCUMENT_PLACEHOLDER_TEXT: &str = "."; +const TOOL_RESULT_IMAGE_PLACEHOLDER: &str = + "[Tool returned an image - see the following user message]"; +const TOOL_USE_ID_MAX_LEN: usize = 64; +const TOOL_USE_ID_HASH_LEN: usize = 8; +const HASH_SAMPLE_BYTES: usize = 64 * 1024; +const BAD_MESSAGE: &str = "Invalid Message "; + +fn bad_request(message: impl Into) -> Error { + Error::InvalidRequest(ErrorDetail::Message(message.into())) +} + +fn role(message: &Value) -> &str { + message + .get("role") + .and_then(Value::as_str) + .unwrap_or_default() +} + +fn is_system(message: &Value) -> bool { + role(message) == "system" +} + +/// `CachePointBlock` for a `cache_control`, or nothing for a model that refuses them. +pub fn cache_point(control: Option<&Value>, model: &BedrockModel<'_>) -> Option { + control.filter(|control| !control.is_null())?; + if !model.accepts_cache_points() { + return None; + } + Some(json!({"cachePoint": cache_point_block(control, model)})) +} + +fn cache_point_block(control: Option<&Value>, model: &BedrockModel<'_>) -> Value { + let mut block = Map::from_iter([("type".to_owned(), json!("default"))]); + if let Some(ttl) = control + .and_then(|control| control.get("ttl")) + .and_then(Value::as_str) + .filter(|ttl| matches!(*ttl, "5m" | "1h")) + && model.cache_ttl_supported() + { + block.insert("ttl".into(), json!(ttl)); + } + Value::Object(block) +} + +/// The leading run of system messages as Converse `system` blocks. +fn system_blocks(leading: &[Value], model: &BedrockModel<'_>) -> Vec { + let mut blocks = Vec::new(); + for message in leading { + match message.get("content") { + Some(Value::String(text)) if !text.is_empty() => { + blocks.push(json!({"text": text})); + blocks.extend(cache_point(message.get("cache_control"), model)); + } + Some(Value::Array(parts)) => { + for part in parts { + let text = part.get("text").and_then(Value::as_str).unwrap_or_default(); + if part.get("type").and_then(Value::as_str) == Some("text") && !text.is_empty() + { + blocks.push(json!({"text": text})); + blocks.extend(cache_point(part.get("cache_control"), model)); + } + } + } + _ => {} + } + } + blocks +} + +fn assistant_has_tool_calls(message: &Value) -> bool { + role(message) == "assistant" && message.get("tool_calls").is_some_and(truthy) +} + +/// `_opens_with_tool_result`. +fn opens_with_tool_result(message: &Value) -> bool { + match role(message) { + "tool" => true, + "user" => { + let first = match message.get("content") { + Some(Value::Array(parts)) => parts.first(), + _ => None, + }; + first + .and_then(|part| part.get("type")) + .and_then(Value::as_str) + == Some("tool_result") + } + _ => false, + } +} + +/// `_reordered_around_tool_results`. +fn reordered_around_tool_results(messages: &[Value], index: usize) -> Vec { + let message = &messages[index]; + let system_run_start = |end: usize| { + (0..end) + .rev() + .find(|j| !is_system(&messages[*j])) + .map_or(0, |j| j + 1) + }; + if opens_with_tool_result(message) { + if index + 1 < messages.len() && opens_with_tool_result(&messages[index + 1]) { + return vec![message.clone()]; + } + let tool_run_start = (0..=index) + .rev() + .find(|j| !opens_with_tool_result(&messages[*j])) + .map_or(0, |j| j + 1); + let run_start = system_run_start(tool_run_start); + let run = &messages[run_start..tool_run_start]; + if !run.is_empty() && run_start >= 1 && assistant_has_tool_calls(&messages[run_start - 1]) { + let mut kept = vec![message.clone()]; + kept.extend(run.iter().cloned()); + return kept; + } + return vec![message.clone()]; + } + if !is_system(message) { + return vec![message.clone()]; + } + let run_start = system_run_start(index); + let run_end = (index..messages.len()) + .find(|j| !is_system(&messages[*j])) + .unwrap_or(messages.len()); + if run_end < messages.len() + && opens_with_tool_result(&messages[run_end]) + && run_start > 0 + && assistant_has_tool_calls(&messages[run_start - 1]) + { + return Vec::new(); + } + vec![message.clone()] +} + +/// `_system_role_message_as_user`: a mid-conversation system message as a user turn. +fn system_as_user(message: &Value) -> Option { + let block = |text: &str, cache_control: Option<&Value>| { + let mut block = Map::from_iter([ + ("type".to_owned(), json!("text")), + ("text".to_owned(), json!(text)), + ]); + if let Some(cache_control) = cache_control.filter(|c| !c.is_null()) { + block.insert("cache_control".into(), cache_control.clone()); + } + Value::Object(block) + }; + let texts: Vec = match message.get("content") { + Some(Value::String(text)) if !text.is_empty() => { + vec![block(text, message.get("cache_control"))] + } + Some(Value::Array(parts)) => parts + .iter() + .filter(|part| part.get("type").and_then(Value::as_str) == Some("text")) + .filter_map(|part| { + let text = part + .get("text") + .and_then(Value::as_str) + .filter(|t| !t.is_empty())?; + Some(block(text, part.get("cache_control"))) + }) + .collect(), + _ => Vec::new(), + }; + if texts.is_empty() { + return None; + } + let mut content = vec![json!({"type": "text", "text": CONVERTED_SYSTEM_NOTE})]; + content.extend(texts); + Some(json!({"role": "user", "content": content})) +} + +/// `_transform_system_message`: the conversation after the leading system run, and that run +/// as `system` blocks. +pub fn transform_system_message( + messages: Vec, + model: &BedrockModel<'_>, +) -> (Vec, Vec) { + let leading = messages + .iter() + .position(|message| !is_system(message)) + .unwrap_or(messages.len()); + let system = system_blocks(&messages[..leading], model); + let remaining = &messages[leading..]; + let kept = (0..remaining.len()) + .flat_map(|index| reordered_around_tool_results(remaining, index)) + .filter_map(|message| match is_system(&message) { + true => system_as_user(&message), + false => Some(message), + }) + .collect(); + (kept, system) +} + +/// `convert_content_list_to_str`. +fn content_text(message: &Value) -> String { + match message.get("content") { + Some(Value::String(text)) => text.clone(), + Some(Value::Array(parts)) => parts + .iter() + .filter_map(|part| part.get("text").and_then(Value::as_str)) + .collect(), + _ => String::new(), + } +} + +/// `_neutralize_orphaned_tool_blocks`: tool calls and results without `tools=` as text. +pub fn neutralize_orphaned_tool_blocks( + messages: Vec, + has_tools: bool, + guardrail_active: bool, +) -> Vec { + let orphaned = messages.iter().any(|message| { + assistant_has_tool_calls(message) || matches!(role(message), "tool" | "function") + }); + if has_tools || !orphaned { + return messages; + } + messages + .into_iter() + .map(|message| { + if assistant_has_tool_calls(&message) { + let base = content_text(&message); + let calls = message + .get("tool_calls") + .and_then(Value::as_array) + .into_iter() + .flatten() + .map(|call| { + let function = call.get("function"); + let name = function + .and_then(|f| f.get("name")) + .and_then(Value::as_str) + .filter(|name| !name.is_empty()) + .unwrap_or("unknown_tool"); + let arguments = function + .and_then(|f| f.get("arguments")) + .and_then(Value::as_str) + .unwrap_or_default(); + let label = match call + .get("id") + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + { + Some(id) => format!("tool call {id}"), + None => "tool call".to_owned(), + }; + format!("[{label}: {name}({arguments})]") + }); + let text: Vec = std::iter::once(base) + .chain(calls) + .filter(|part| !part.is_empty()) + .collect(); + return json!({"role": "assistant", "content": text.join("\n")}); + } + if matches!(role(&message), "tool" | "function") { + let target = ["tool_call_id", "name"] + .iter() + .find_map(|key| { + message + .get(*key) + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + }) + .unwrap_or("unknown"); + let rendered = content_text(&message).trim().to_owned(); + let rendered = match rendered.is_empty() { + true => "".to_owned(), + false => rendered, + }; + let text = format!("[tool result for {target}: {rendered}]"); + return match guardrail_active { + true => { + json!({"role": "user", "content": [{"type": "guarded_text", "text": text}]}) + } + false => json!({"role": "user", "content": text}), + }; + } + message + }) + .collect() +} + +/// `_convert_consecutive_user_messages_to_guarded_text`. +pub fn guard_trailing_user_messages(mut messages: Vec) -> Vec { + for message in messages.iter_mut().rev() { + if role(message) != "user" { + break; + } + match message.get_mut("content") { + Some(Value::Array(parts)) => { + if parts + .iter() + .any(|part| part.get("type").and_then(Value::as_str) == Some("guarded_text")) + { + continue; + } + for part in parts.iter_mut() { + if part.get("type").and_then(Value::as_str) == Some("text") { + let text = part.get("text").cloned().unwrap_or(Value::Null); + *part = json!({"type": "guarded_text", "text": text}); + } + } + } + Some(Value::String(text)) => { + let text = text.clone(); + message["content"] = json!([{"type": "guarded_text", "text": text}]); + } + _ => {} + } + } + messages +} + +/// `_sanitize_bedrock_tool_use_id`. +pub fn sanitize_tool_use_id(id: &str) -> String { + let sanitized: String = id + .chars() + .map( + |c| match c.is_ascii_alphanumeric() || matches!(c, '_' | '.' | ':' | '-') { + true => c, + false => '_', + }, + ) + .collect(); + if sanitized == id && sanitized.len() <= TOOL_USE_ID_MAX_LEN { + return sanitized; + } + let digest = hex(&Sha256::digest(id.as_bytes())); + let head: String = sanitized + .chars() + .take(TOOL_USE_ID_MAX_LEN - TOOL_USE_ID_HASH_LEN - 1) + .collect(); + format!("{head}_{}", &digest[..TOOL_USE_ID_HASH_LEN]) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|byte| format!("{byte:02x}")).collect() +} + +/// `mimetypes.guess_all_extensions` for the document types Converse takes. +fn document_format(mime_type: &str) -> Result { + let format = match mime_type { + "application/pdf" => "pdf", + "text/csv" => "csv", + "application/msword" => "doc", + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" => "docx", + "application/vnd.ms-excel" => "xls", + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet" => "xlsx", + "text/html" => "html", + "text/plain" => "txt", + "text/markdown" | "text/x-markdown" => "md", + _ => { + return Err(bad_request(format!( + "No supported extensions for MIME type: {mime_type}. Supported formats: {}", + python_list(&SUPPORTED_DOCUMENT_FORMATS) + ))); + } + }; + Ok(format.to_owned()) +} + +fn python_list(items: &[&str]) -> String { + format!( + "[{}]", + items + .iter() + .map(|item| format!("'{item}'")) + .collect::>() + .join(", ") + ) +} + +fn is_document_mime(mime_type: &str) -> bool { + mime_type.starts_with("application") || mime_type.starts_with("text") +} + +/// `BedrockImageProcessor.validate_format`. +fn validate_format(mime_type: &str, image_format: &str) -> Result { + if is_document_mime(mime_type) { + return document_format(mime_type); + } + let supported: Vec<&str> = SUPPORTED_VIDEO_FORMATS + .iter() + .chain(SUPPORTED_IMAGE_FORMATS.iter()) + .copied() + .collect(); + if !supported.contains(&image_format) { + return Err(bad_request(format!( + "Unsupported image format: {image_format}. Supported formats: {}", + python_list(&supported) + ))); + } + Ok(image_format.to_owned()) +} + +/// The name Python derives from a document's content: a hash of its first 64 KB and length. +fn document_hash(data: &str) -> String { + let normalized: String = data.split_whitespace().collect(); + let bytes = normalized.as_bytes(); + let mut hasher = Sha256::new(); + hasher.update(&bytes[..bytes.len().min(HASH_SAMPLE_BYTES)]); + hasher.update(bytes.len().to_string().as_bytes()); + hex(&hasher.finalize())[..16].to_owned() +} + +/// `BedrockImageProcessor.process_image_sync` over a URL the host already inlined. +fn media_block(url: &str, format: Option<&str>) -> Result { + if !url.contains("base64") { + return Err(match url.contains("http://") || url.contains("https://") { + true => Error::Unsupported("a media URL the host did not download"), + false => bad_request( + "Unsupported image type. Expected either image url or base64 encoded string", + ), + }); + } + let (metadata, data) = url.split_once(',').unwrap_or((url, "")); + let (mut mime_type, mut image_format) = match metadata + .strip_prefix("data:") + .and_then(|rest| rest.split(";base64").next()) + { + Some(mime) => { + let mime = mime.split(';').next().unwrap_or(mime).to_owned(); + let format = mime.split('/').nth(1).unwrap_or_default().to_owned(); + (mime, format) + } + None => ("image/jpeg".to_owned(), "jpeg".to_owned()), + }; + if let Some(format) = format.filter(|format| !format.is_empty()) { + mime_type = format.to_owned(); + image_format = mime_type.split('/').nth(1).unwrap_or_default().to_owned(); + } + let image_format = validate_format(&mime_type, &image_format)?; + let source = json!({"bytes": data}); + if is_document_mime(&mime_type) { + let name = format!("DocumentPDFmessages_{}_{image_format}", document_hash(data)); + return Ok(json!({"document": {"source": source, "format": image_format, "name": name}})); + } + if SUPPORTED_VIDEO_FORMATS + .iter() + .any(|video| image_format.starts_with(video)) + { + return Ok(json!({"video": {"source": source, "format": image_format}})); + } + Ok(json!({"image": {"source": source, "format": image_format}})) +} + +fn image_url_block(part: &Value) -> Result { + let image_url = part.get("image_url"); + let (url, format) = match image_url { + Some(Value::Object(image)) => ( + image.get("url").and_then(Value::as_str).unwrap_or_default(), + image.get("format").and_then(Value::as_str), + ), + Some(Value::String(url)) => (url.as_str(), None), + _ => ("", None), + }; + media_block(url, format) +} + +/// `process_file_message`. +fn file_block(part: &Value) -> Result { + let Some(file) = part.get("file").filter(|file| !file.is_null()) else { + return Err(bad_request( + "Content block has type='file' but is missing the required 'file' field", + )); + }; + let source = ["file_id", "file_data"].iter().find_map(|key| { + file.get(*key) + .and_then(Value::as_str) + .filter(|v| !v.is_empty()) + }); + let Some(source) = source else { + return Err(bad_request(format!( + "file_data and file_id cannot both be None. Got={part}" + ))); + }; + media_block(source, file.get("format").and_then(Value::as_str)) +} + +/// `_process_document_message`. +fn document_block(part: &Value) -> Result { + let source = part.get("source").cloned().unwrap_or(Value::Null); + let source_type = source + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + if source_type != "base64" { + return Err(bad_request(format!( + "Bedrock Converse only supports base64-encoded document sources, got '{source_type}'. Please convert the document to base64 before sending to Bedrock." + ))); + } + let media_type = source + .get("media_type") + .and_then(Value::as_str) + .unwrap_or_default(); + let data = source + .get("data") + .and_then(Value::as_str) + .unwrap_or_default(); + let format = validate_format(media_type, media_type.split('/').nth(1).unwrap_or_default())?; + let name = format!("Document_{}_{format}", document_hash(data)); + Ok(json!({"document": {"source": {"bytes": data}, "format": format, "name": name}})) +} + +/// `skip_empty_text_blocks` for a list: whitespace-only text parts go. +fn non_empty_parts(parts: &[Value]) -> Vec { + parts + .iter() + .filter(|part| { + !(part.get("type").and_then(Value::as_str) == Some("text") + && part + .get("text") + .and_then(Value::as_str) + .unwrap_or_default() + .trim() + .is_empty()) + }) + .cloned() + .collect() +} + +fn user_blocks(message: &Value, model: &BedrockModel<'_>) -> Result, Error> { + let mut blocks = Vec::new(); + match message.get("content") { + Some(Value::Array(parts)) => { + for part in non_empty_parts(parts) { + if !part.is_object() { + continue; + } + let text = || part.get("text").cloned().unwrap_or(Value::Null); + match part.get("type").and_then(Value::as_str) { + Some("text" | "grounding_source" | "query") => { + blocks.push(json!({"text": text()})) + } + Some("guarded_text") => { + blocks.push(json!({"guardContent": {"text": {"text": text()}}})) + } + Some("image_url") => blocks.push(image_url_block(&part)?), + Some("file") => blocks.push(file_block(&part)?), + Some("document") => blocks.push(document_block(&part)?), + _ => {} + } + blocks.extend(cache_point(part.get("cache_control"), model)); + } + } + Some(Value::String(text)) if !text.is_empty() => { + blocks.push(json!({"text": text})); + blocks.extend(cache_point(message.get("cache_control"), model)); + } + _ => {} + } + Ok(blocks) +} + +/// `_parse_bedrock_tool_result_content_list` item by item. +fn tool_result_parts(content: &[Value]) -> Result, Error> { + let mut parts = Vec::new(); + for item in content { + match item.get("type").and_then(Value::as_str) { + Some("text") => { + parts.push(json!({"text": item.get("text").cloned().unwrap_or(Value::Null)})) + } + Some(kind @ ("image_url" | "file")) => { + let block = match kind { + "image_url" => image_url_block(item)?, + _ => file_block(item)?, + }; + for key in ["image", "document"] { + if let Some(media) = block.get(key) { + parts.push(json!({key: media})); + } + } + } + Some("tool_reference") => { + if let Some(name) = item.get("tool_name").and_then(Value::as_str) { + parts.push(json!({"text": name})); + } + } + _ => {} + } + } + Ok(parts) +} + +/// `_convert_to_bedrock_tool_call_result`. +fn tool_result(message: &Value) -> Result { + let search_results: Vec = message + .get("search_results") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|result| result.is_object()) + .map(|result| json!({"searchResult": result})) + .collect(); + let used_search_results = !search_results.is_empty(); + let content = match used_search_results { + true => search_results, + false => match message.get("content") { + Some(Value::String(text)) => vec![json!({"text": text})], + Some(Value::Array(items)) => tool_result_parts(items)?, + _ => Vec::new(), + }, + }; + let id = match message.get("tool_call_id") { + Some(Value::String(id)) => id.clone(), + Some(Value::Null) => "None".to_owned(), + Some(other) => other.to_string(), + None => uuid::Uuid::new_v4().to_string(), + }; + let mut result = Map::from_iter([ + ("content".to_owned(), Value::Array(content)), + ("toolUseId".to_owned(), json!(sanitize_tool_use_id(&id))), + ]); + if used_search_results { + result.insert("status".into(), json!("success")); + } + Ok(json!({"toolResult": result})) +} + +/// `split_concatenated_json_objects`. +fn concatenated_objects(raw: &str) -> Vec { + serde_json::Deserializer::from_str(raw.trim()) + .into_iter::() + .map_while(Result::ok) + .map(|value| match value { + Value::Object(_) => value, + _ => json!({}), + }) + .collect() +} + +/// `_convert_to_bedrock_tool_call_invoke`. +fn tool_uses(calls: &[Value], model: &BedrockModel<'_>) -> Result, Error> { + let mut blocks = Vec::new(); + for call in calls.iter().filter(|call| call.get("function").is_some()) { + let id = call.get("id").and_then(Value::as_str).unwrap_or_default(); + let function = &call["function"]; + let name = make_valid_bedrock_tool_name( + function + .get("name") + .and_then(Value::as_str) + .unwrap_or_default(), + ); + let arguments = function + .get("arguments") + .and_then(Value::as_str) + .unwrap_or_default(); + let cache = cache_point(call.get("cache_control"), model); + if arguments.trim().is_empty() { + blocks.push(json!({"toolUse": {"input": {}, "name": name, "toolUseId": sanitize_tool_use_id(id)}})); + } else { + match serde_json::from_str::(arguments) { + Ok(parsed) => { + let input = match parsed { + Value::Object(_) => parsed, + _ => json!({}), + }; + blocks.push(json!({"toolUse": {"input": input, "name": name, "toolUseId": sanitize_tool_use_id(id)}})); + } + Err(_) => { + let objects = concatenated_objects(arguments); + if objects.is_empty() { + blocks.push(json!({"toolUse": {"input": {}, "name": name, "toolUseId": sanitize_tool_use_id(id)}})); + } else { + for (index, object) in objects.into_iter().enumerate() { + let block_id = match index { + 0 => id.to_owned(), + _ => format!("{id}_{index}"), + }; + blocks.push(json!({"toolUse": {"input": object, "name": name, "toolUseId": sanitize_tool_use_id(&block_id)}})); + } + } + } + } + } + blocks.extend(cache); + } + Ok(blocks) +} + +/// `translate_thinking_blocks_to_reasoning_content_blocks` and +/// `add_thinking_blocks_to_assistant_content`. +fn add_thinking_blocks(blocks: &[Value], to_claude: bool, out: &mut Vec) { + let mut reasoning = Vec::new(); + for block in blocks { + if to_claude && crate::anthropic::chat::messages::is_encrypted_reasoning(block) { + continue; + } + let thinking = block + .get("thinking") + .and_then(Value::as_str) + .unwrap_or_default(); + let data = block + .get("data") + .and_then(Value::as_str) + .filter(|d| !d.trim().is_empty()); + if block.get("type").and_then(Value::as_str) == Some("redacted_thinking") + && to_claude + && let Some(data) = data + { + use base64::Engine; + let content = base64::engine::general_purpose::STANDARD + .encode(native_redacted_thinking_data(data).as_bytes()); + reasoning.push(json!({"reasoningContent": {"redactedContent": content}})); + continue; + } + match block.get("signature").filter(|s| !s.is_null()) { + Some(signature) if !signature.as_str().is_some_and(str::is_empty) => reasoning.push( + json!({"reasoningContent": {"reasoningText": {"text": thinking, "signature": signature}}}), + ), + _ => { + if !thinking.trim().is_empty() { + out.push(json!({"text": thinking})); + } + } + } + } + out.extend(reasoning); +} + +fn assistant_blocks(message: &Value, model: &BedrockModel<'_>) -> Result, Error> { + let mut blocks = Vec::new(); + let to_claude = model.accepts_redacted_thinking(); + if let Some(Value::Array(thinking)) = message.get("thinking_blocks") { + add_thinking_blocks(thinking, to_claude, &mut blocks); + } + match message.get("content") { + Some(Value::Array(parts)) => { + let parts = non_empty_parts(parts); + let mut converted = Vec::new(); + for part in parts.iter().filter(|part| part.is_object()) { + match part.get("type").and_then(Value::as_str) { + Some("thinking" | "redacted_thinking") => { + add_thinking_blocks(std::slice::from_ref(part), to_claude, &mut converted) + } + Some("text") => { + let text = part.get("text").and_then(Value::as_str).unwrap_or_default(); + if !text.trim().is_empty() { + converted.push(json!({"text": text})); + } + } + Some("image_url") => converted.push(image_url_block(part)?), + _ => {} + } + converted.extend(cache_point(part.get("cache_control"), model)); + } + blocks.extend(converted); + } + Some(Value::String(text)) => { + let non_content = message.get("tool_calls").is_some_and(truthy) + || message.get("function_call").is_some_and(truthy); + if !text.trim().is_empty() { + blocks.push(json!({"text": text})); + } + // An all-whitespace string becomes `None` when the message carries tool calls, + // and a `None` content adds no cache point. + if !(text.trim().is_empty() && non_content) { + blocks.extend(cache_point(message.get("cache_control"), model)); + } + } + _ => {} + } + if let Some(Value::Array(calls)) = message.get("tool_calls") + && !calls.is_empty() + { + blocks.extend(tool_uses(calls, model)?); + } + Ok(blocks) +} + +/// `_deduplicate_bedrock_content_blocks`. +fn dedupe(blocks: Vec, key: &str) -> Vec { + let mut seen: HashSet = HashSet::new(); + blocks + .into_iter() + .filter(|block| { + match block + .get(key) + .and_then(|keyed| keyed.get("toolUseId")) + .and_then(Value::as_str) + .filter(|id| !id.is_empty()) + { + Some(id) => seen.insert(id.to_owned()), + None => true, + } + }) + .collect() +} + +/// `_sort_bedrock_assistant_content_blocks`. +fn sorted_assistant_blocks(mut blocks: Vec) -> Vec { + blocks.sort_by_key(|block| { + if block.get("reasoningContent").is_some() { + 0 + } else if block.get("toolUse").is_some() || block.get("cachePoint").is_some() { + 2 + } else { + 1 + } + }); + blocks +} + +fn push_user_turn(contents: &mut Vec, blocks: Vec) { + if let Some(last) = contents.last_mut() + && last["role"] == "user" + && let Some(Value::Array(existing)) = last.get_mut("content") + { + existing.extend(blocks); + return; + } + contents.push(json!({"role": "user", "content": blocks})); +} + +/// `_bedrock_converse_messages_pt`. +pub fn converse_messages( + messages: &[Value], + model: &BedrockModel<'_>, +) -> Result, Error> { + if messages.is_empty() { + return Err(bad_request(format!( + "{BAD_MESSAGE}bedrock requires at least one non-system message" + ))); + } + let mut contents: Vec = Vec::new(); + let mut index = 0; + while index < messages.len() { + let start = index; + let mut user_content = Vec::new(); + while index < messages.len() && role(&messages[index]) == "user" { + user_content.extend(user_blocks(&messages[index], model)?); + index += 1; + } + if !user_content.is_empty() { + push_user_turn(&mut contents, user_content); + } + let mut tool_content = Vec::new(); + while index < messages.len() && role(&messages[index]) == "tool" { + let message = &messages[index]; + tool_content.push(tool_result(message)?); + let control = message + .get("cache_control") + .filter(|c| !c.is_null()) + .or_else(|| { + message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .find_map(|part| part.get("cache_control").filter(|c| !c.is_null())) + }); + tool_content.extend(cache_point(control, model)); + index += 1; + } + let tool_content = dedupe(tool_content, "toolResult"); + if !tool_content.is_empty() { + push_user_turn(&mut contents, tool_content); + } + let mut assistant_content = Vec::new(); + while index < messages.len() && role(&messages[index]) == "assistant" { + assistant_content.extend(assistant_blocks(&messages[index], model)?); + index += 1; + } + let assistant_content = sorted_assistant_blocks(dedupe(assistant_content, "toolUse")); + if !assistant_content.is_empty() { + contents.push(json!({"role": "assistant", "content": assistant_content})); + } + if index == start { + return Err(bad_request(format!( + "{BAD_MESSAGE}passed in {}", + crate::base_llm::chat::params::python_dumps(&messages[index]) + ))); + } + } + rename_duplicate_documents(&mut contents); + let mut contents: Vec = contents + .into_iter() + .map(with_text_when_document_only) + .collect(); + if model.accepts_redacted_thinking() + && let Some(last) = contents.last_mut() + && last["role"] == "assistant" + && let Some(Value::Array(blocks)) = last.get_mut("content") + && let Some(position) = blocks.iter().rposition(|block| block.get("text").is_some()) + { + let text = blocks[position]["text"] + .as_str() + .unwrap_or_default() + .trim_end() + .to_owned(); + blocks[position] = json!({"text": text}); + } + if !model.supports_tool_result_images() { + contents = contents + .into_iter() + .map(images_beside_tool_results) + .collect(); + } + Ok(contents) +} + +/// `_rename_duplicate_bedrock_document_names`. +fn rename_duplicate_documents(contents: &mut [Value]) { + let names = |contents: &[Value]| -> Vec { + contents + .iter() + .flat_map(|message| message["content"].as_array().cloned().unwrap_or_default()) + .filter_map(|block| { + block + .get("document")? + .get("name")? + .as_str() + .map(str::to_owned) + }) + .filter(|name| !name.is_empty()) + .collect() + }; + let mut used: HashSet = names(contents).into_iter().collect(); + let mut counts: std::collections::HashMap = std::collections::HashMap::new(); + for message in contents.iter_mut() { + let Some(Value::Array(blocks)) = message.get_mut("content") else { + continue; + }; + for block in blocks { + let Some(Value::Object(document)) = block.get_mut("document") else { + continue; + }; + let Some(name) = document + .get("name") + .and_then(Value::as_str) + .map(str::to_owned) + else { + continue; + }; + if name.is_empty() { + continue; + } + let count = counts.entry(name.clone()).or_insert(0); + *count += 1; + if *count > 1 { + let mut suffix = *count; + let mut renamed = format!("{name}_{suffix}"); + while used.contains(&renamed) { + suffix += 1; + renamed = format!("{name}_{suffix}"); + } + used.insert(renamed.clone()); + document.insert("name".into(), json!(renamed)); + } + } + } +} + +/// `_with_text_when_document_only`. +fn with_text_when_document_only(message: Value) -> Value { + let blocks = message["content"].as_array().cloned().unwrap_or_default(); + let needs_text = message["role"] == "user" + && blocks.iter().any(|block| block.get("document").is_some()) + && blocks.iter().all(|block| block.get("text").is_none()); + if !needs_text { + return message; + } + let cut = match blocks + .last() + .is_some_and(|block| block.get("cachePoint").is_some()) + { + true => blocks.len() - 1, + false => blocks.len(), + }; + let mut content = blocks[..cut].to_vec(); + content.push(json!({"text": DOCUMENT_PLACEHOLDER_TEXT})); + content.extend(blocks[cut..].iter().cloned()); + json!({"role": "user", "content": content}) +} + +/// `place_tool_result_images` for a model that refuses an image inside a tool result. +fn images_beside_tool_results(message: Value) -> Value { + let Some(blocks) = message["content"].as_array() else { + return message; + }; + let mut rewritten = Vec::new(); + let mut changed = false; + for block in blocks { + let Some(result) = block.get("toolResult") else { + rewritten.push(block.clone()); + continue; + }; + let parts = result["content"].as_array().cloned().unwrap_or_default(); + let images: Vec = parts + .iter() + .filter_map(|part| part.get("image").cloned()) + .collect(); + if images.is_empty() { + rewritten.push(block.clone()); + continue; + } + changed = true; + let mut kept: Vec = parts + .into_iter() + .filter(|part| part.get("image").is_none()) + .collect(); + if kept.is_empty() { + kept.push(json!({"text": TOOL_RESULT_IMAGE_PLACEHOLDER})); + } + let mut result = result.clone(); + result["content"] = Value::Array(kept); + rewritten.push(json!({"toolResult": result})); + rewritten.extend(images.into_iter().map(|image| json!({"image": image}))); + } + match changed { + true => json!({"role": message["role"].clone(), "content": rewritten}), + false => message, + } +} + +/// `normalize_json_schema_custom_types_to_object`. +fn normalize_custom_types(schema: &mut Value) { + let Value::Object(node) = schema else { return }; + if node.get("type").and_then(Value::as_str) == Some("custom") { + node.insert("type".into(), json!("object")); + } + for key in ["items", "additionalProperties"] { + if let Some(child @ Value::Object(_)) = node.get_mut(key) { + normalize_custom_types(child); + } + } + for key in ["properties", "$defs", "definitions"] { + if let Some(Value::Object(children)) = node.get_mut(key) { + children.values_mut().for_each(normalize_custom_types); + } + } + for key in ["anyOf", "allOf", "oneOf"] { + if let Some(Value::Array(children)) = node.get_mut(key) { + children.iter_mut().for_each(normalize_custom_types); + } + } +} + +/// `_bedrock_tools_pt`, with the names it rewrote mapped back for the response. +pub fn converse_tools( + tools: &[Value], + model: &BedrockModel<'_>, +) -> Result<(Vec, std::collections::HashMap), Error> { + let strict_supported = model.supports_strict_tools(); + let mut blocks = Vec::new(); + let mut renamed = std::collections::HashMap::new(); + for (index, tool) in tools.iter().enumerate() { + let Some(object) = tool.as_object() else { + continue; + }; + if ["systemTool", "toolSpec", "cachePoint"] + .iter() + .any(|key| object.contains_key(*key)) + { + blocks.push(tool.clone()); + continue; + } + let kind = object + .get("type") + .and_then(Value::as_str) + .unwrap_or_default(); + if kind.starts_with("computer_") { + return Err(Error::Unsupported( + "a computer use tool on Bedrock Converse", + )); + } + if matches!( + kind, + "tool_search_tool_regex_20251119" | "tool_search_tool_bm25_20251119" + ) { + continue; + } + if !object.contains_key("function") && !object.contains_key("input_schema") { + continue; + } + let default_schema = json!({"type": "object", "properties": {}}); + let (mut parameters, raw_name, description) = + if object.contains_key("input_schema") && !object.contains_key("function") { + ( + object + .get("input_schema") + .filter(|v| truthy(v)) + .cloned() + .unwrap_or(default_schema), + object + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + object.get("description").cloned(), + ) + } else { + let function = object.get("function").cloned().unwrap_or(Value::Null); + ( + function + .get("parameters") + .cloned() + .unwrap_or(default_schema), + function + .get("name") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(), + function.get("description").cloned(), + ) + }; + let raw_name = match raw_name.trim().is_empty() { + true => format!("litellm_unnamed_tool_{index}"), + false => raw_name, + }; + let name = make_valid_bedrock_tool_name(&raw_name); + if name != raw_name { + renamed.insert(name.clone(), raw_name.clone()); + } + let description = match description { + Some(Value::String(text)) if !text.is_empty() => json!(text), + Some(other) if truthy(&other) => other, + _ => json!(name), + }; + let defs = match parameters.as_object_mut().and_then(|p| p.remove("$defs")) { + Some(Value::Object(defs)) => defs, + _ => Map::new(), + }; + unpack_defs(&mut parameters, &defs); + normalize_custom_types(&mut parameters); + let valid_roots = [ + "array", "boolean", "integer", "null", "number", "object", "string", + ]; + if !parameters + .get("type") + .and_then(Value::as_str) + .is_some_and(|kind| valid_roots.contains(&kind)) + && let Value::Object(object) = &mut parameters + { + object.insert("type".into(), json!("object")); + } + let mut json_schema = Map::from_iter([ + ( + "type".to_owned(), + parameters.get("type").cloned().unwrap_or(Value::Null), + ), + ( + "properties".to_owned(), + parameters + .get("properties") + .cloned() + .unwrap_or_else(|| json!({})), + ), + ( + "required".to_owned(), + parameters + .get("required") + .cloned() + .unwrap_or_else(|| json!([])), + ), + ]); + if strict_supported + && let Some(additional) = parameters + .get("additionalProperties") + .filter(|v| !v.is_null()) + { + json_schema.insert("additionalProperties".into(), additional.clone()); + } + let mut spec = Map::from_iter([ + ("inputSchema".to_owned(), json!({"json": json_schema})), + ("name".to_owned(), json!(name)), + ("description".to_owned(), description), + ]); + let strict = object.get("function").and_then(|f| f.get("strict")); + if strict_supported && strict.is_some_and(truthy) { + spec.insert("strict".into(), strict.cloned().unwrap_or(Value::Null)); + } + blocks.push(json!({"toolSpec": spec})); + if let Some(control) = object.get("cache_control").filter(|c| !c.is_null()) + && model.accepts_cache_points() + && control + .get("type") + .and_then(Value::as_str) + .unwrap_or("ephemeral") + == "ephemeral" + { + blocks.push(json!({"cachePoint": cache_point_block(Some(control), model)})); + } + } + Ok((blocks, renamed)) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + use crate::base_llm::chat::params::ChatRequestContext; + + fn model() -> BedrockModel<'static> { + BedrockModel::new( + "anthropic.claude-sonnet-4-5-v1:0", + &ChatRequestContext { + drop_params: Some(false), + model_info: None, + }, + ) + } + + #[rstest] + #[case("call_1", "call_1")] + #[case("call 1", "call_1_")] + fn tool_use_ids_are_sanitized(#[case] id: &str, #[case] prefix: &str) { + let sanitized = sanitize_tool_use_id(id); + assert!(sanitized.starts_with(prefix)); + if id != prefix { + assert_eq!(sanitized.len(), prefix.len() + TOOL_USE_ID_HASH_LEN); + } + } + + #[rstest] + fn a_tool_round_trip_becomes_tool_use_and_tool_result() { + let messages = vec![ + json!({"role": "user", "content": "weather?"}), + json!({"role": "assistant", "content": "checking", "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "get.weather", "arguments": "{\"city\":\"Paris\"}"}} + ]}), + json!({"role": "tool", "tool_call_id": "c1", "content": "sunny"}), + ]; + let converted = converse_messages(&messages, &model()).unwrap(); + assert_eq!( + converted, + vec![ + json!({"role": "user", "content": [{"text": "weather?"}]}), + json!({"role": "assistant", "content": [ + {"text": "checking"}, + {"toolUse": {"input": {"city": "Paris"}, "name": "get_weather", "toolUseId": "c1"}} + ]}), + json!({"role": "user", "content": [{"toolResult": {"content": [{"text": "sunny"}], "toolUseId": "c1"}}]}), + ] + ); + } + + #[rstest] + fn images_and_documents_become_blocks() { + let messages = vec![json!({"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": "data:image/png;base64,AAAA"}}, + {"type": "file", "file": {"file_data": "data:application/pdf;base64,JVBE"}, "cache_control": {"type": "ephemeral"}} + ]})]; + let converted = converse_messages(&messages, &model()).unwrap(); + let content = converted[0]["content"].as_array().unwrap(); + assert_eq!( + content[0], + json!({"image": {"source": {"bytes": "AAAA"}, "format": "png"}}) + ); + assert_eq!(content[1]["document"]["format"], "pdf"); + assert!( + content[1]["document"]["name"] + .as_str() + .unwrap() + .starts_with("DocumentPDFmessages_") + ); + assert_eq!(content[2], json!({"text": "."})); + assert_eq!(content[3], json!({"cachePoint": {"type": "default"}})); + } + + #[rstest] + fn system_messages_become_system_blocks_and_operator_notes() { + let (messages, system) = transform_system_message( + vec![ + json!({"role": "system", "content": "be terse", "cache_control": {"type": "ephemeral"}}), + json!({"role": "user", "content": "hi"}), + json!({"role": "system", "content": "later"}), + ], + &model(), + ); + assert_eq!( + system, + vec![ + json!({"text": "be terse"}), + json!({"cachePoint": {"type": "default"}}) + ] + ); + assert_eq!(messages[1]["content"][0]["text"], CONVERTED_SYSTEM_NOTE); + assert_eq!(messages[1]["content"][1]["text"], "later"); + } + + #[rstest] + fn orphaned_tool_blocks_become_text() { + let messages = neutralize_orphaned_tool_blocks( + vec![ + json!({"role": "assistant", "tool_calls": [{"id": "c", "function": {"name": "f", "arguments": "{}"}}]}), + json!({"role": "tool", "tool_call_id": "c", "content": "ok"}), + ], + false, + false, + ); + assert_eq!( + messages[0], + json!({"role": "assistant", "content": "[tool call c: f({})]"}) + ); + assert_eq!( + messages[1], + json!({"role": "user", "content": "[tool result for c: ok]"}) + ); + } + + #[rstest] + fn tools_become_tool_specs() { + let (tools, renamed) = converse_tools( + &[json!({"type": "function", "function": {"name": "get.weather", "parameters": { + "type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"] + }}})], + &model(), + ) + .unwrap(); + assert_eq!( + tools, + vec![ + json!({"toolSpec": {"inputSchema": {"json": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}, "name": "get_weather", "description": "get_weather"}}) + ] + ); + assert_eq!(renamed["get_weather"], "get.weather"); + } +} diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_params.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_params.rs new file mode 100644 index 00000000000..04c44d8f16c --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_params.rs @@ -0,0 +1,1039 @@ +//! Python's `AmazonConverseConfig.get_supported_openai_params` and `map_openai_params`, and +//! the Bedrock model-id reads behind them. + +use serde_json::{Map, Value, json}; + +use crate::{ + Error, ErrorDetail, + anthropic::chat::params::{ + ModelCapabilities, apply_sampling_param, cap_thinking_budget, map_context_management, + map_reasoning_effort, output_config_effort, python_str, translate_legacy_thinking, + unmapped_reasoning_effort, + }, + base_llm::chat::{ + params::{ + ChatRequestContext, DEFAULT_MAX_TOKENS, DropPolicy, RESPONSE_FORMAT_TOOL_NAME, + capability, is_thinking_enabled, model_field, ordered, python_dumps, unsupported_value, + }, + response_format::ResponseFormatSchema, + }, +}; + +/// `get_bedrock_cross_region_inference_regions`. +const CROSS_REGION_PREFIXES: [&str; 7] = ["global", "us", "eu", "apac", "jp", "au", "us-gov"]; + +/// `BEDROCK_MIN_THINKING_BUDGET_TOKENS`. +const MIN_THINKING_BUDGET_TOKENS: i64 = 1024; + +/// `BEDROCK_OPENAI_COMPAT_MIN_MAX_TOKENS`. +const OPENAI_COMPAT_MIN_MAX_TOKENS: i64 = 16; + +/// `_BEDROCK_OUTPUT_CONFIG_EFFORT_ORDER`. +const OUTPUT_CONFIG_EFFORT_ORDER: [&str; 5] = ["low", "medium", "high", "xhigh", "max"]; + +/// `strip_bedrock_routing_prefix`: each prefix in turn, as Python applies them. +pub fn strip_bedrock_routing_prefix(model: &str) -> String { + let mut model = model.to_owned(); + for prefix in [ + "bedrock/", + "chat_completions/", + "converse/", + "invoke/", + "openai/", + "mantle/", + "nova-2/", + "nova/", + ] { + if model.starts_with(prefix) { + model = model + .split_once('/') + .map(|(_, rest)| rest.to_owned()) + .unwrap_or_default(); + } + } + model +} + +/// `strip_bedrock_throughput_suffix`: `::k` loses its throughput and `[...]` is cut. +fn strip_throughput_suffix(model: &str) -> String { + let mut model = model.to_owned(); + if let Some(head) = model.strip_suffix('k') + && let Some((rest, digits)) = head.rsplit_once(':') + && !digits.is_empty() + && digits.chars().all(|c| c.is_ascii_digit()) + && let Some((_, minor)) = rest.rsplit_once(':') + && !minor.is_empty() + && minor.chars().all(|c| c.is_ascii_digit()) + { + model = rest.to_owned(); + } + if model.ends_with(']') + && let Some(open) = model.rfind('[') + && model[open + 1..model.len() - 1] + .chars() + .all(|c| c.is_alphanumeric() || c == '_') + && open + 1 < model.len() - 1 + { + model.truncate(open); + } + model +} + +/// Whether `segment` reads as an AWS region, such as `us-east-1` or `us-gov-west-1`. +fn is_aws_region(segment: &str) -> bool { + let parts: Vec<&str> = segment.split('-').collect(); + parts.len() >= 3 + && parts[0].len() == 2 + && parts[0].chars().all(|c| c.is_ascii_lowercase()) + && parts + .last() + .is_some_and(|last| !last.is_empty() && last.chars().all(|c| c.is_ascii_digit())) + && parts[1..parts.len() - 1] + .iter() + .all(|part| !part.is_empty() && part.chars().all(|c| c.is_ascii_lowercase())) +} + +/// `get_bedrock_base_model`. +pub fn bedrock_base_model(model: &str) -> String { + let mut stripped = model; + for prefix in ["bedrock/converse/", "bedrock/", "converse/"] { + if let Some(rest) = stripped.strip_prefix(prefix) { + stripped = rest; + break; + } + } + if stripped.starts_with("nova-2/") { + return "amazon.nova-2-custom".into(); + } + if stripped.starts_with("nova/") { + return "amazon.nova-custom".into(); + } + let mut model = strip_bedrock_routing_prefix(model); + if model.to_lowercase().contains("arn") { + model = model.rsplit('/').next().unwrap_or_default().to_owned(); + } + let model = strip_throughput_suffix(&model); + if let Some((region, rest)) = model.split_once('.') + && CROSS_REGION_PREFIXES.contains(®ion) + { + return rest.to_owned(); + } + if let Some((region, rest)) = model.split_once('/') + && is_aws_region(region) + { + return rest.to_owned(); + } + model +} + +/// `_is_nova_2_model`. +fn is_nova_2_model(model: &str) -> bool { + let mut stripped = model; + for prefix in ["bedrock/converse/", "bedrock/", "converse/"] { + if let Some(rest) = stripped.strip_prefix(prefix) { + stripped = rest; + break; + } + } + for prefix in ["us.", "eu.", "apac."] { + if let Some(rest) = stripped.strip_prefix(prefix) { + stripped = rest; + break; + } + } + stripped.starts_with("amazon.nova-2-") || stripped.starts_with("nova-2/") +} + +/// `openai.gpt-` anywhere in the id. +fn is_openai_gpt_reasoning_model(model: &str) -> bool { + model.match_indices("openai.gpt-").any(|(at, _)| { + model[at + "openai.gpt-".len()..] + .chars() + .next() + .is_some_and(|c| c.is_ascii_digit()) + }) +} + +/// `xai.grok-` after a start, `.` or `/`. +fn is_xai_grok_model(model: &str) -> bool { + model + .match_indices("xai.grok-") + .any(|(at, _)| at == 0 || matches!(model.as_bytes()[at - 1], b'.' | b'/')) +} + +/// `openai.gpt-[.]` after a start, `.` or `/`. +fn openai_gpt_version(model: &str) -> Option<(u32, u32)> { + model.match_indices("openai.").find_map(|(at, _)| { + if at != 0 && !matches!(model.as_bytes()[at - 1], b'.' | b'/') { + return None; + } + let rest = model[at..].strip_prefix("openai.gpt-")?; + let major: String = rest.chars().take_while(char::is_ascii_digit).collect(); + if major.is_empty() || major.len() > 3 { + return None; + } + let after = &rest[major.len()..]; + let minor = after + .strip_prefix('.') + .map(|tail| { + tail.chars() + .take_while(char::is_ascii_digit) + .collect::() + }) + .filter(|minor| !minor.is_empty() && minor.len() <= 3) + .unwrap_or_default(); + Some((major.parse().ok()?, minor.parse().unwrap_or(0))) + }) +} + +/// `bedrock_rejects_stop_sequences`. +fn rejects_stop_sequences(model: &str) -> bool { + is_xai_grok_model(model) || openai_gpt_version(model).is_some_and(|version| version >= (5, 6)) +} + +/// The model a Converse request is for, and the cost-map reads its rules make. +pub struct BedrockModel<'a> { + pub model: String, + pub base: String, + pub context: ChatRequestContext<'a>, + pub caps: ModelCapabilities<'a>, +} + +impl<'a> BedrockModel<'a> { + pub fn new(model: &str, context: &ChatRequestContext<'a>) -> Self { + Self { + model: model.to_owned(), + base: bedrock_base_model(model), + context: *context, + caps: ModelCapabilities::new(model, context), + } + } + + pub fn is_anthropic(&self) -> bool { + self.base.starts_with("anthropic") + } + + /// The keys Python's provider-aware lookups resolve `model` through. + pub fn candidates(&self) -> Vec { + let stripped = strip_bedrock_routing_prefix(&self.model); + let mut candidates = vec![ + self.model.clone(), + format!("bedrock/{}", self.model), + format!("bedrock_converse/{}", self.model), + stripped.clone(), + self.base.clone(), + format!("bedrock/{}", self.base), + ]; + candidates.dedup(); + candidates + } + + pub fn flag(&self, key: &str) -> Result, Error> { + capability(&self.context, &self.candidates(), key) + } + + pub fn supports(&self, key: &str) -> Result { + Ok(self.flag(key)? == Some(true)) + } + + /// `_supports_sampling_params`. + pub fn supports_sampling(&self) -> Result { + if self.is_anthropic() { + return Ok(true); + } + let mut keys = vec![self.model.clone()]; + keys.extend(["global.", "us.", "eu."].map(|prefix| format!("{prefix}{}", self.base))); + Ok(capability(&self.context, &keys, "supports_sampling_params")?.unwrap_or(true)) + } + + fn is_deepseek(&self) -> bool { + self.model.contains("deepseek") || self.base.contains("deepseek") + } + + fn is_deepseek_r1(&self) -> bool { + self.model.contains("deepseek.r1") || self.base.contains("deepseek.r1") + } + + /// `_model_accepts_anthropic_thinking_param`. + fn accepts_anthropic_thinking(&self) -> Result { + if self.is_deepseek() { + return Ok(false); + } + Ok(self.model.contains("claude-3-7") + || self.model.contains("claude-sonnet-4") + || self.model.contains("claude-opus-4") + || self.supports("supports_reasoning")?) + } + + /// `bedrock_converse_supports_parallel_tool_use_config`. + pub fn supports_parallel_tool_use_config(&self) -> Result { + capability( + &self.context, + &[self.model.clone(), self.base.clone()], + "supports_parallel_tool_use_config", + ) + .map(|flag| flag == Some(true)) + } + + /// `bedrock_model_accepts_cache_points`. + pub fn accepts_cache_points(&self) -> bool { + let openai_family = self + .model + .match_indices("openai.") + .any(|(at, _)| at == 0 || matches!(self.model.as_bytes()[at - 1], b'.' | b'/')); + if openai_family { + return false; + } + let Some(info) = self.context.model_info else { + return true; + }; + let map_keys = [self.model.clone(), self.base.clone()]; + let registered = ["", "converse/", "converse_like/"] + .map(|route| format!("bedrock/{route}{}", self.model)); + let explicit = registered + .iter() + .chain(map_keys.iter()) + .filter_map(|key| info.entry(key)) + .find_map(|entry| { + entry + .get("supports_prompt_cache_breakpoint") + .filter(|flag| !flag.is_null()) + .map(|flag| flag == &Value::Bool(true)) + }); + if let Some(explicit) = explicit { + return explicit; + } + let entries: Vec<_> = map_keys.iter().filter_map(|key| info.entry(key)).collect(); + entries.is_empty() + || entries + .iter() + .any(|entry| entry.get("supports_prompt_caching") == Some(&Value::Bool(true))) + } + + /// `supports_bedrock_prompt_cache_ttl`. + pub fn cache_ttl_supported(&self) -> bool { + model_field( + &self.context, + &[self.model.clone(), self.base.clone()], + "cache_creation_input_token_cost_above_1hr", + ) + .is_some() + } + + /// `bedrock_converse_supports_strict_tools`. + pub fn supports_strict_tools(&self) -> bool { + if !self.is_anthropic() { + return false; + } + self.context + .model_info + .and_then(|info| { + [self.base.clone(), format!("bedrock/{}", self.base)] + .iter() + .filter_map(|key| info.entry(key)) + .find_map(|entry| { + entry + .get("bedrock_converse_supports_strict_tools") + .and_then(Value::as_bool) + }) + }) + .unwrap_or(true) + } + + /// `bedrock_converse_supports_tool_result_images`. + pub fn supports_tool_result_images(&self) -> bool { + let Some(info) = self.context.model_info else { + return true; + }; + let keys = [ + self.model.clone(), + strip_bedrock_routing_prefix(&self.model), + ]; + keys.iter().filter_map(|key| info.entry(key)).all(|entry| { + entry.get("supports_bedrock_converse_tool_result_images") != Some(&Value::Bool(false)) + }) + } + + /// `bedrock_converse_accepts_redacted_thinking` for a model id that names its model. + pub fn accepts_redacted_thinking(&self) -> bool { + self.base.starts_with("anthropic") || self.base.starts_with("claude") + } + + /// `normalize_bedrock_opus_output_config_effort`: an effort above the model's ceiling + /// comes down to it. + pub fn normalize_output_config_effort(&self, output_config: &mut Map) { + let Some(effort) = output_config.get("effort").and_then(Value::as_str) else { + return; + }; + let Some(rank) = OUTPUT_CONFIG_EFFORT_ORDER.iter().position(|e| *e == effort) else { + return; + }; + let ceiling = model_field( + &self.context, + &self.candidates(), + "bedrock_output_config_effort_ceiling", + ); + let Some(ceiling) = ceiling.as_ref().and_then(Value::as_str) else { + return; + }; + if let Some(ceiling_rank) = OUTPUT_CONFIG_EFFORT_ORDER + .iter() + .position(|e| *e == ceiling) + && rank > ceiling_rank + { + output_config.insert("effort".into(), json!(ceiling)); + } + } + + /// `_validate_anthropic_adaptive_effort`. + pub fn validate_adaptive_effort(&self, effort: &Value) -> Result<(), Error> { + let text = effort + .as_str() + .filter(|effort| OUTPUT_CONFIG_EFFORT_ORDER.contains(effort)); + let Some(text) = text else { + let shown = match effort { + Value::String(text) => format!("'{text}'"), + other => python_dumps(other), + }; + return Err(unsupported_value(format!( + "Invalid reasoning_effort/output_config.effort value: {shown}. Must be one of: 'low', 'medium', 'high', 'xhigh', or 'max'." + ))); + }; + let refused = match text { + "max" + if !(self.caps.is_adaptive()? + || self.caps.supports("supports_max_reasoning_effort")?) => + { + Some("max") + } + "xhigh" if !self.caps.supports("supports_xhigh_reasoning_effort")? => Some("xhigh"), + _ => None, + }; + match refused { + Some(value) => Err(Error::InvalidRequest(ErrorDetail::UnsupportedValue { + field: "effort", + value, + model: self.model.clone(), + })), + None => Ok(()), + } + } + + /// `AnthropicConfig.model_supports_effort_param` under `bedrock`. + pub fn supports_effort_param(&self) -> Result { + if self.caps.supports("supports_output_config")? { + return Ok(true); + } + for level in ["low", "minimal", "medium", "high", "xhigh", "max"] { + if self + .caps + .supports(&format!("supports_{level}_reasoning_effort"))? + { + return Ok(true); + } + } + Ok(false) + } +} + +/// `get_supported_openai_params`. +pub fn supported_params(model: &BedrockModel<'_>) -> Result, Error> { + let mut params = vec![ + "max_tokens", + "max_completion_tokens", + "stream", + "stream_options", + "stop", + ]; + if model.supports_sampling()? { + params.extend(["temperature", "top_p"]); + } + params.extend([ + "extra_headers", + "response_format", + "requestMetadata", + "service_tier", + "parallel_tool_calls", + ]); + if model.model.contains("arn") { + params.extend([ + "tools", + "tool_choice", + "thinking", + "reasoning_effort", + "output_config", + ]); + if model.model.to_lowercase().contains("nova") { + params.push("web_search_options"); + } + return Ok(params); + } + let base = model.base.as_str(); + if [ + "anthropic", + "mistral", + "cohere", + "meta.llama3-1", + "meta.llama3-2", + "meta.llama3-3", + "meta.llama4", + "amazon.nova", + ] + .iter() + .any(|family| base.starts_with(family)) + || model.supports("supports_function_calling")? + { + params.push("tools"); + } + if base.starts_with("amazon.nova") { + params.push("web_search_options"); + } + if model.supports("supports_tool_choice")? { + params.push("tool_choice"); + } + if model.model.contains("gpt-oss") + || is_openai_gpt_reasoning_model(&model.model) + || is_openai_gpt_reasoning_model(base) + { + params.push("reasoning_effort"); + } else if model.is_deepseek() { + if !model.is_deepseek_r1() { + params.push("reasoning_effort"); + } + } else if is_nova_2_model(&model.model) { + params.push("reasoning_effort"); + } else if model.accepts_anthropic_thinking()? { + params.extend(["thinking", "reasoning_effort", "output_config"]); + } + if model.is_anthropic() { + params.push("context_management"); + } + Ok(params) +} + +/// `map_tool_choice_values`. +pub fn map_tool_choice_values( + model: &BedrockModel<'_>, + tool_choice: &Value, + drop: DropPolicy, +) -> Result, Error> { + // `forced_tool_use_downgraded`: refused without `drop_params`, `auto` with it. + let downgraded = || -> Result { + if !model.caps.forced_tool_use_unsupported()? { + return Ok(false); + } + drop.unsupported(|| forced_tool_use_refusal(&model.model))?; + Ok(true) + }; + match tool_choice { + Value::String(choice) if choice == "none" => { + drop.unsupported(|| { + unsupported_value( + "Bedrock doesn't support tool_choice=none. To drop it from the call, set `litellm.drop_params = True.", + ) + })?; + Ok(None) + } + Value::String(choice) if choice == "required" => Ok(Some(match downgraded()? { + true => json!({"auto": {}}), + false => json!({"any": {}}), + })), + Value::String(choice) if choice == "auto" => Ok(Some(json!({"auto": {}}))), + Value::Object(choice) => { + if downgraded()? { + return Ok(Some(json!({"auto": {}}))); + } + let name = choice + .get("function") + .and_then(|function| function.get("name")) + .and_then(Value::as_str) + .unwrap_or_default(); + Ok(Some( + json!({"tool": {"name": make_valid_bedrock_tool_name(name)}}), + )) + } + other => Err(unsupported_value(format!( + "Bedrock doesn't support tool_choice={}. Supported tool_choice values=['auto', 'required', json object]. To drop it from the call, set `litellm.drop_params = True.", + python_str(other) + ))), + } +} + +fn forced_tool_use_refusal(model: &str) -> Error { + unsupported_value(format!( + "{model} does not support forced tool use (tool_choice='required' or a named tool). Use tool_choice='auto' and tell the model in the prompt when to call the tool, or set `litellm.drop_params = True` to downgrade to 'auto' automatically." + )) +} + +/// `make_valid_bedrock_tool_name`: `[a-zA-Z][a-zA-Z0-9_-]*`. +pub fn make_valid_bedrock_tool_name(name: &str) -> String { + let Some(first) = name.chars().next() else { + return String::new(); + }; + let prefixed = match first.is_alphabetic() { + true => name.to_owned(), + false => format!("a{name}"), + }; + prefixed + .chars() + .map(|c| match c.is_alphanumeric() || c == '_' || c == '-' { + true => c, + false => '_', + }) + .collect() +} + +/// `_add_additional_properties_to_schema`. +fn with_closed_objects(schema: &Value) -> Value { + let Value::Object(object) = schema else { + return schema.clone(); + }; + let mut result = object.clone(); + if result.get("type").and_then(Value::as_str) == Some("object") + && !result.contains_key("additionalProperties") + { + result.insert("additionalProperties".into(), json!(false)); + } + for key in ["properties", "$defs", "definitions"] { + if let Some(Value::Object(children)) = result.get(key).cloned() { + let closed: Map = children + .iter() + .map(|(name, child)| (name.clone(), with_closed_objects(child))) + .collect(); + result.insert(key.into(), Value::Object(closed)); + } + } + if let Some(items @ Value::Object(_)) = result.get("items").cloned() { + result.insert("items".into(), with_closed_objects(&items)); + } + for key in ["anyOf", "allOf", "oneOf"] { + if let Some(Value::Array(items)) = result.get(key).cloned() { + result.insert( + key.into(), + Value::Array(items.iter().map(with_closed_objects).collect()), + ); + } + } + Value::Object(result) +} + +/// `_create_output_config_for_response_format`. +pub fn output_config_for_response_format( + schema: Option<&Value>, + name: Option<&Value>, + description: Option<&Value>, +) -> Value { + let schema_text = match schema { + Some(schema) => python_dumps(&with_closed_objects(schema)), + None => "{}".to_owned(), + }; + let mut json_schema = Map::from_iter([("schema".to_owned(), json!(schema_text))]); + if let Some(name) = name.filter(|v| !v.is_null()) { + json_schema.insert("name".into(), name.clone()); + } + if let Some(description) = description.filter(|v| !v.is_null()) { + json_schema.insert("description".into(), description.clone()); + } + json!({"textFormat": {"type": "json_schema", "structure": {"jsonSchema": json_schema}}}) +} + +/// What `map_openai_params` hands to the request build. +#[derive(Debug, Default)] +pub struct MappedParams { + pub params: Map, + pub json_mode: bool, + /// The schema of the `json_tool_call` tool a `response_format` added. + pub response_format_schema: Option, +} + +fn add_tools(out: &mut Map, tools: Vec) { + match out.get_mut("tools") { + Some(Value::Array(existing)) => existing.extend(tools), + _ => { + out.insert("tools".into(), Value::Array(tools)); + } + } +} + +/// `_handle_reasoning_effort_parameter`. +fn map_reasoning_effort_param( + out: &mut Map, + model: &BedrockModel<'_>, + effort: &str, +) -> Result<(), Error> { + if model.model.contains("gpt-oss") || model.model.contains("deepseek") { + out.insert("reasoning_effort".into(), json!(effort)); + } else if is_openai_gpt_reasoning_model(&model.model) { + out.insert("reasoning".into(), json!({"effort": effort})); + } else if is_nova_2_model(&model.model) { + if !matches!(effort, "low" | "medium" | "high") { + return Err(unsupported_value(format!( + "Invalid reasoning_effort value '{effort}' for Nova 2 models. Supported values: ['low', 'medium', 'high']" + ))); + } + out.insert( + "reasoningConfig".into(), + json!({"type": "enabled", "maxReasoningEffort": effort}), + ); + } else { + match map_reasoning_effort(effort, &model.caps)? { + None => { + out.remove("thinking"); + out.remove("output_config"); + } + Some(thinking) => { + out.insert("thinking".into(), thinking); + if model.caps.is_adaptive()? { + let mapped = output_config_effort(effort) + .ok_or_else(|| unmapped_reasoning_effort(effort))?; + let mut output_config = match out.get("output_config") { + Some(Value::Object(existing)) => existing.clone(), + _ => Map::new(), + }; + output_config + .entry("effort") + .or_insert_with(|| json!(mapped)); + model.normalize_output_config_effort(&mut output_config); + if let Some(effort) = output_config.get("effort") { + model.validate_adaptive_effort(effort)?; + } + out.insert("output_config".into(), Value::Object(output_config)); + out.insert("_output_config_normalized".into(), json!(true)); + } + } + } + } + Ok(()) +} + +/// `AmazonConverseConfig.map_openai_params`. +pub fn map_openai_params( + openai: &Map, + model: &BedrockModel<'_>, + drop: DropPolicy, +) -> Result { + let thinking_requested = is_thinking_enabled(openai); + let mut out = Map::new(); + let mut json_mode = false; + let mut response_format_schema = None; + for (name, value) in ordered(openai) { + match name { + "response_format" => { + let Some(format) = value.as_object() else { + continue; + }; + if format.get("type").and_then(Value::as_str) == Some("text") { + continue; + } + let (schema, schema_name, description) = match format.get("response_schema") { + Some(schema) => (Some(schema.clone()), None, None), + None => match format.get("json_schema") { + Some(json_schema) => ( + json_schema.get("schema").cloned(), + json_schema.get("name").cloned(), + json_schema.get("description").cloned(), + ), + None => (None, None, None), + }, + }; + let schema = schema.filter(|schema| !schema.is_null()); + if schema.is_some() && model.supports("supports_native_structured_output")? { + out.insert( + "outputConfig".into(), + output_config_for_response_format( + schema.as_ref(), + schema_name.as_ref(), + description.as_ref(), + ), + ); + } else if let Some(schema) = schema { + let mut function = Map::from_iter([ + ("name".to_owned(), json!(RESPONSE_FORMAT_TOOL_NAME)), + ("parameters".to_owned(), schema.clone()), + ]); + if let Some(description) = + description.filter(|d| d.as_str().is_some_and(|d| !d.is_empty())) + { + function.insert("description".into(), description); + } + add_tools( + &mut out, + vec![json!({"type": "function", "function": function})], + ); + response_format_schema = ResponseFormatSchema::from_schema(&schema); + if model.supports("supports_tool_choice")? + && !thinking_requested + && !model.caps.forced_tool_use_unsupported()? + { + out.insert( + "tool_choice".into(), + json!({"tool": {"name": RESPONSE_FORMAT_TOOL_NAME}}), + ); + } + } + json_mode = true; + } + "max_tokens" | "max_completion_tokens" => { + let requires_min = is_openai_gpt_reasoning_model(&model.model) + || model.model.contains("xai.grok-"); + let value = match value.as_i64() { + Some(tokens) if requires_min && value.is_i64() => { + json!(tokens.max(OPENAI_COMPAT_MIN_MAX_TOKENS)) + } + _ => value.clone(), + }; + out.insert("maxTokens".into(), value); + } + "stream" => { + out.insert("stream".into(), value.clone()); + } + "stop" if !rejects_stop_sequences(&model.model) => { + let stop = match value { + Value::String(text) if text.is_empty() => continue, + Value::String(text) => json!([text]), + other => other.clone(), + }; + out.insert("stopSequences".into(), stop); + } + "temperature" | "top_p" => { + let output_key = match name { + "top_p" => "topP", + _ => name, + }; + if model.is_anthropic() { + apply_sampling_param(&mut out, &model.caps, drop, name, value, output_key)?; + } else if !model.supports_sampling()? { + drop.unsupported(|| { + unsupported_value(format!( + "{} does not support {name}={}. To drop unsupported params, set `litellm.drop_params = True`.", + model.model, + python_str(value) + )) + })?; + } else { + out.insert(output_key.into(), value.clone()); + } + } + "tools" => { + let Some(tools) = value.as_array() else { + continue; + }; + if tools + .iter() + .any(|tool| tool.get("type").and_then(Value::as_str) == Some("tool_search")) + { + return Err(Error::Unsupported("the OpenAI tool_search tool on Bedrock")); + } + add_tools(&mut out, tools.clone()); + } + "tool_choice" => { + if let Some(choice) = map_tool_choice_values(model, value, drop)? { + out.insert("tool_choice".into(), choice); + } + } + "parallel_tool_calls" => { + out.insert( + "_parallel_tool_use_config".into(), + json!({"tool_choice": {"type": "auto", "disable_parallel_tool_use": !crate::base_llm::chat::params::truthy(value)}}), + ); + } + "thinking" if model.is_deepseek() => {} + "thinking" if !is_openai_gpt_reasoning_model(&model.model) => { + let adaptive_request = + value.get("type").and_then(Value::as_str) == Some("adaptive"); + if adaptive_request && !model.caps.is_adaptive()? { + let max_tokens = openai + .get("max_completion_tokens") + .filter(|v| crate::base_llm::chat::params::truthy(v)) + .or_else(|| openai.get("max_tokens")) + .filter(|v| !v.is_null()); + let legacy = map_reasoning_effort("medium", &model.caps)?; + if let Some(capped) = + legacy.and_then(|legacy| cap_thinking_budget(legacy, max_tokens)) + { + out.insert("thinking".into(), capped); + } + } else { + out.insert("thinking".into(), value.clone()); + translate_legacy_thinking(&mut out, &model.caps, drop)?; + } + } + "reasoning_effort" => { + let Some(effort) = value.as_str() else { + continue; + }; + if model.is_deepseek_r1() { + continue; + } + if is_openai_gpt_reasoning_model(&model.model) { + return Err(Error::Unsupported( + "reasoning_effort for an OpenAI GPT model on Bedrock Converse", + )); + } + map_reasoning_effort_param(&mut out, model, effort)?; + } + "context_management" => { + if let Some(mapped) = match value { + Value::Array(_) => map_context_management(value), + Value::Object(_) => Some(value.clone()), + _ => None, + } { + out.insert("context_management".into(), mapped); + } + } + "service_tier" => { + let Some(tier) = value.as_str() else { continue }; + let tier = match tier { + "auto" => "default", + other => other, + }; + if matches!(tier, "default" | "flex" | "priority") { + out.insert("serviceTier".into(), json!({"type": tier})); + } + } + "web_search_options" + if value.is_object() && model.model.to_lowercase().contains("nova") => + { + add_tools( + &mut out, + vec![json!({"systemTool": {"name": "nova_grounding"}})], + ); + } + _ => {} + } + } + if !model.model.contains("gpt-oss") && !is_nova_2_model(&model.model) { + if let Some(Value::Object(thinking)) = out.get_mut("thinking") + && let Some(budget) = thinking.get("budget_tokens").and_then(Value::as_i64) + && thinking.get("budget_tokens").is_some_and(Value::is_i64) + && budget < MIN_THINKING_BUDGET_TOKENS + { + thinking.insert("budget_tokens".into(), json!(MIN_THINKING_BUDGET_TOKENS)); + } + let max_tokens_in_request = + openai.contains_key("max_tokens") || openai.contains_key("max_completion_tokens"); + if is_thinking_enabled(&out) + && !max_tokens_in_request + && let Some(budget) = out + .get("thinking") + .and_then(|thinking| thinking.get("budget_tokens")) + .and_then(Value::as_i64) + { + out.insert( + "maxTokens".into(), + json!(budget + DEFAULT_MAX_TOKENS as i64), + ); + } + } + if is_thinking_enabled(&out) + && out + .get("tool_choice") + .and_then(Value::as_object) + .is_some_and(|choice| choice.contains_key("any") || choice.contains_key("tool")) + { + out.insert("tool_choice".into(), json!({"auto": {}})); + } + Ok(MappedParams { + params: out, + json_mode, + response_format_schema, + }) +} + +/// `_validate_request_metadata`. +pub fn validate_request_metadata(metadata: &Value) -> Result<(), Error> { + let Some(metadata) = metadata.as_object() else { + return Err(unsupported_value("requestMetadata must be a dictionary")); + }; + if metadata.len() > 16 { + return Err(unsupported_value( + "requestMetadata can contain a maximum of 16 items", + )); + } + let allowed = + |c: char| c.is_ascii_alphanumeric() || c.is_whitespace() || ":_@$#=/+,.-".contains(c); + for (key, value) in metadata { + let Some(value) = value.as_str() else { + return Err(unsupported_value("requestMetadata values must be strings")); + }; + if key.is_empty() || key.chars().count() > 256 { + return Err(unsupported_value( + "requestMetadata key length must be 1-256 characters", + )); + } + if value.chars().count() > 256 { + return Err(unsupported_value( + "requestMetadata value length must be 0-256 characters", + )); + } + if !key.chars().all(allowed) { + return Err(unsupported_value(format!( + "requestMetadata key '{key}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]" + ))); + } + if !value.chars().all(allowed) { + return Err(unsupported_value(format!( + "requestMetadata value '{value}' contains invalid characters. Allowed: [a-zA-Z0-9\\s:_@$#=/+,.-]" + ))); + } + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case( + "us.meta.llama3-2-11b-instruct-v1:0", + "meta.llama3-2-11b-instruct-v1:0" + )] + #[case( + "bedrock/converse/anthropic.claude-3-5-sonnet-20240620-v1:0", + "anthropic.claude-3-5-sonnet-20240620-v1:0" + )] + #[case( + "anthropic.claude-3-5-sonnet-20241022-v2:0:51k", + "anthropic.claude-3-5-sonnet-20241022-v2:0" + )] + #[case("bedrock/nova-2/arn:aws:bedrock:x", "amazon.nova-2-custom")] + #[case( + "arn:aws:bedrock:us-east-1:1:inference-profile/us.anthropic.claude-x", + "anthropic.claude-x" + )] + #[case("us-west-2/mistral.mistral-large", "mistral.mistral-large")] + #[case("global.anthropic.claude-sonnet-4-5", "anthropic.claude-sonnet-4-5")] + fn base_models_match_python(#[case] model: &str, #[case] expected: &str) { + assert_eq!(bedrock_base_model(model), expected); + } + + #[rstest] + #[case("get weather", "get_weather")] + #[case("1tool", "a1tool")] + #[case("", "")] + #[case("ok-name_1", "ok-name_1")] + fn tool_names_are_made_valid(#[case] name: &str, #[case] expected: &str) { + assert_eq!(make_valid_bedrock_tool_name(name), expected); + } + + #[rstest] + fn gpt_versions_parse() { + assert_eq!(openai_gpt_version("openai.gpt-5.6"), Some((5, 6))); + assert!(rejects_stop_sequences("us.xai.grok-4")); + assert!(!rejects_stop_sequences("openai.gpt-oss-20b-1:0")); + } + + #[rstest] + fn request_metadata_is_validated() { + assert!(validate_request_metadata(&json!({"a": "b"})).is_ok()); + assert_eq!( + validate_request_metadata(&json!({"a": 1})) + .unwrap_err() + .to_string(), + "invalid request: requestMetadata values must be strings" + ); + } +} diff --git a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs index a27a71b2802..5bcfb02e730 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/converse_transformation.rs @@ -1,3 +1,6 @@ +//! Python's `AmazonConverseConfig` for `/chat/completions`: `get_optional_params` and +//! `_transform_request` on the way in, `_transform_response` on the way out. + use litellm_auth::AwsParams; use litellm_auth::{CredentialPlacement, SecretValue}; use litellm_auth_aws::{ @@ -5,51 +8,51 @@ use litellm_auth_aws::{ constants::{AWS_BEARER_TOKEN_BEDROCK, BEDROCK_RUNTIME_ENDPOINT_TEMPLATE, BEDROCK_SERVICE}, resolve_bedrock_region, }; -use litellm_core_utils::{ - core_helpers::{finish_reason_for, unix_now, usage_from_parts}, - prompt_templates::factory::{Conversation, TurnRole, build_conversation}, -}; +use litellm_core_utils::core_helpers::{finish_reason_for, unix_now, usage_from_parts}; use litellm_llms_types::{ formats::chat_completions::{ - ChatCompletionsChoice, ChatCompletionsChoiceMessage, ChatCompletionsResponse, - ChatCompletionsUsage, ChatMessage, ChatMessageContent, + ChatCompletionThinkingBlock, ChatCompletionToolCallChunk, + ChatCompletionToolCallFunctionChunk, ChatCompletionsChoice, ChatCompletionsChoiceMessage, + ChatCompletionsResponse, ChatCompletionsUsage, ChatMessage, }, - providers::bedrock::{CONVERSE_PATH, ConverseResponse}, + providers::bedrock::{CONVERSE_PATH, ConverseUsage}, }; +use serde::Deserialize; use serde_json::{Map, Value, json}; use crate::{ - Error, + Error, ErrorDetail, + anthropic::chat::{messages::native_redacted_thinking_data, params::apply_sampling_param}, base_llm::{ auth::AuthScheme, chat::{ + media::{data_url, remote_urls}, + params::{ + ChatRequestContext, RESPONSE_FORMAT_TOOL_NAME, python_dumps, sort_params, truthy, + }, streaming::{ChatStream, StreamShape}, transformation::{ BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, - Unsupported, ValidatedEnvironment, unsupported_message, unsupported_param, + Unsupported, ValidatedEnvironment, }, }, }, - bedrock::chat::invoke_handler::converse_chat_stream, + bedrock::chat::{ + converse_messages::{ + converse_messages, converse_tools, guard_trailing_user_messages, + neutralize_orphaned_tool_blocks, transform_system_message, + }, + converse_params::{ + BedrockModel, MappedParams, map_openai_params, output_config_for_response_format, + supported_params, validate_request_metadata, + }, + invoke_handler::converse_chat_stream, + }, }; /// The action a streaming Converse call goes to, in place of [`CONVERSE_PATH`]. const CONVERSE_STREAM_PATH: &str = "/converse-stream"; -/// Converse parameter names, post `map_openai_params`, that the Rust path can -/// place verbatim in `inferenceConfig`. -/// -/// `topK` is deliberately absent: Python routes it to -/// `additionalModelRequestFields` for Anthropic base models and to -/// `inferenceConfig` otherwise, and that branch reads the model catalog the -/// core cannot see. -const SUPPORTED_PARAMS: &[(&str, &str)] = &[ - ("max_tokens", "maxTokens"), - ("temperature", "temperature"), - ("top_p", "topP"), - ("stop", "stopSequences"), -]; - const AWS_BEDROCK_RUNTIME_ENDPOINT: &str = "aws_bedrock_runtime_endpoint"; /// AWS call configuration a host passes down: consumed for signing and endpoint @@ -68,6 +71,30 @@ const CONFIG_PARAMS: &[&str] = &[ AWS_BEDROCK_RUNTIME_ENDPOINT, ]; +/// `AmazonConverseConfig.__annotations__` plus `top_k`, the tool params and the config +/// blocks: what stays out of `additionalModelRequestFields`. +const INFERENCE_PARAMS: [&str; 6] = [ + "maxTokens", + "stopSequences", + "temperature", + "topP", + "topK", + "top_k", +]; +const TOOL_PARAMS: [&str; 2] = ["tools", "tool_choice"]; +const CONFIG_BLOCKS: [&str; 3] = ["guardrailConfig", "performanceConfig", "serviceTier"]; +/// `filter_internal_params`. +const INTERNAL_PARAMS: [&str; 3] = [ + "skip_mcp_handler", + "mcp_handler_context", + "_skip_mcp_handler", +]; + +const FINE_GRAINED_TOOL_STREAMING_BETA: &str = "fine-grained-tool-streaming-2025-05-14"; +const EFFORT_BETA: &str = "effort-2025-11-24"; +const COMPACT_BETA: &str = "compact-2026-01-12"; +const COMPACT_EDIT_TYPE: &str = "compact_20260112"; + enum ConverseStopReason { Value(String), Unknown(String), @@ -92,10 +119,552 @@ impl ConverseStopReason { } } +#[derive(Deserialize)] +struct ResponseProjection { + output: OutputProjection, + #[serde(default)] + usage: ConverseUsage, + #[serde(rename = "stopReason")] + stop_reason: Option, +} + +#[derive(Deserialize)] +struct OutputProjection { + message: Option, +} + +#[derive(Deserialize)] +struct MessageProjection { + content: Vec, +} + +#[derive(Deserialize)] +struct ContentProjection { + #[serde(default)] + text: Option, + #[serde(default, rename = "toolUse")] + tool_use: Option, + #[serde(default, rename = "reasoningContent")] + reasoning_content: Option, + #[serde(default, rename = "citationsContent")] + citations_content: Option, +} + pub struct AmazonConverseConfig; pub const BEDROCK_CHAT_COMPLETIONS_CONFIG: AmazonConverseConfig = AmazonConverseConfig; +/// `parse_content_for_reasoning`: a leading `` block split from the answer. +fn parse_content_for_reasoning(text: &str) -> (Option, String) { + for tag in ["think", "thinking", "budget:thinking"] { + let Some(rest) = text.strip_prefix(&format!("<{tag}>")) else { + continue; + }; + let closes = ["think", "thinking", "budget:thinking"] + .iter() + .filter_map(|close| { + let marker = format!(""); + rest.find(&marker).map(|at| (at, marker.len())) + }) + .min_by_key(|(at, _)| *at); + if let Some((at, len)) = closes { + return (Some(rest[..at].to_owned()), rest[at + len..].to_owned()); + } + } + (None, text.to_owned()) +} + +/// `_unwrap_bedrock_properties`: a lone `properties` key gives up its value. +pub fn unwrap_bedrock_properties(text: &str) -> String { + match serde_json::from_str::(text) { + Ok(Value::Object(object)) if object.len() == 1 && object.contains_key("properties") => { + python_dumps(&object["properties"]) + } + _ => text.to_owned(), + } +} + +/// `eager_input_streaming_flag` over a chat tool. +fn eager_input_streaming(tool: &Value) -> bool { + tool.get("eager_input_streaming") == Some(&Value::Bool(true)) + || tool + .get("function") + .and_then(|function| function.get("eager_input_streaming")) + == Some(&Value::Bool(true)) +} + +/// `_merge_parallel_tool_use_config`. +fn merge_parallel_tool_use_config(additional: &mut Map, config: Map) { + for (key, value) in config { + let merged = match (additional.get(&key), &value) { + (Some(Value::Object(existing)), Value::Object(incoming)) => { + let mut merged = incoming.clone(); + merged.extend(existing.iter().map(|(k, v)| (k.clone(), v.clone()))); + merged.extend( + incoming + .iter() + .filter(|(k, _)| k.as_str() != "type") + .map(|(k, v)| (k.clone(), v.clone())), + ); + Value::Object(merged) + } + _ => value, + }; + additional.insert(key, merged); + } +} + +impl AmazonConverseConfig { + fn build_request( + &self, + model: &str, + messages: Vec, + optional_params: Map, + context: &ChatRequestContext<'_>, + ) -> Result { + let bedrock = BedrockModel::new(model, context); + let sorted = sort_params( + optional_params, + CONFIG_PARAMS, + context, + |_| supported_params(&bedrock), + "bedrock", + model, + )?; + let drop = sorted.drop; + let MappedParams { + mut params, + json_mode, + response_format_schema, + } = map_openai_params(&sorted.openai, &bedrock, drop)?; + for name in &sorted.allowed { + if let Some(value) = sorted.openai.get(name) + && !params.contains_key(name) + { + params.insert(name.clone(), value.clone()); + } + } + params.extend(sorted.provider); + + let messages: Vec = messages + .iter() + .map(|message| { + serde_json::to_value(message).map(|mut value| { + if value.get("role").and_then(Value::as_str) == Some("developer") { + value["role"] = json!("system"); + } + value + }) + }) + .collect::>() + .map_err(|error| Error::InvalidRequest(ErrorDetail::invalid("messages", error)))?; + let (messages, system) = transform_system_message(messages, &bedrock); + let guardrail = params.contains_key("guardrailConfig"); + let has_tools = params.get("tools").is_some_and(truthy); + let mut messages = neutralize_orphaned_tool_blocks(messages, has_tools, guardrail); + if guardrail { + messages = guard_trailing_user_messages(messages); + } + + // `maybe_drop_disabled_thinking`. + if params + .get("thinking") + .and_then(|thinking| thinking.get("type")) + .and_then(Value::as_str) + == Some("disabled") + && bedrock.caps.supports("thinking_always_on")? + { + params.remove("thinking"); + } + + // `_prepare_request_params`. + let normalized = params + .remove("_output_config_normalized") + .is_some_and(|flag| truthy(&flag)); + for internal in ["json_mode", "stream", "fake_stream"] { + params.remove(internal); + } + let mut anthropic_output_config = match params.remove("output_config") { + Some(Value::Object(config)) => Some(config), + _ => None, + }; + let mut output_config_format = None; + if let Some(config) = &mut anthropic_output_config { + if let Some(Value::Object(format)) = config.remove("format") { + output_config_format = Some(format); + } + if config.is_empty() { + anthropic_output_config = None; + } + } + let request_metadata = params.remove("requestMetadata").filter(|v| !v.is_null()); + if let Some(metadata) = &request_metadata { + validate_request_metadata(metadata)?; + } + let mut output_config = params.remove("outputConfig").filter(|v| !v.is_null()); + if output_config.is_none() + && let Some(format) = &output_config_format + && format.get("type").and_then(Value::as_str) == Some("json_schema") + && bedrock.is_anthropic() + && bedrock.supports("supports_native_structured_output")? + { + output_config = Some(output_config_for_response_format( + format.get("schema"), + format.get("name"), + format.get("description"), + )); + } + let (mut inference, mut additional): (Map, Map) = + params.into_iter().partition(|(key, _)| { + INFERENCE_PARAMS.contains(&key.as_str()) + || TOOL_PARAMS.contains(&key.as_str()) + || CONFIG_BLOCKS.contains(&key.as_str()) + }); + if let Some(Value::Object(config)) = additional.remove("_parallel_tool_use_config") + && bedrock.supports_parallel_tool_use_config()? + { + merge_parallel_tool_use_config(&mut additional, config); + } + additional.remove("parallel_tool_calls"); + additional.remove("client_metadata"); + let top_k = inference + .remove("topK") + .or_else(|| inference.remove("top_k")) + .filter(|top_k| !top_k.is_null()); + if let Some(top_k) = top_k { + if bedrock.is_anthropic() { + apply_sampling_param( + &mut additional, + &bedrock.caps, + drop, + "top_k", + &top_k, + "top_k", + )?; + } else if bedrock.base.starts_with("amazon.nova") { + additional.insert("inferenceConfig".into(), json!({"topK": top_k})); + } + } + for internal in INTERNAL_PARAMS { + additional.remove(internal); + } + if let Some(mut config) = anthropic_output_config { + if model.contains(":application-inference-profile/") { + additional.insert("output_config".into(), Value::Object(config)); + } else if bedrock.is_anthropic() + && !(drop.drops() && !bedrock.supports_effort_param()?) + { + if !normalized { + bedrock.normalize_output_config_effort(&mut config); + } + if let Some(effort) = config.get("effort").filter(|effort| !effort.is_null()) { + bedrock.validate_adaptive_effort(effort)?; + } + additional.insert("output_config".into(), Value::Object(config)); + } + } + + // `_process_tools_and_beta`. + let tools = match inference.remove("tools") { + Some(Value::Array(tools)) => tools, + _ => Vec::new(), + }; + let (tool_blocks, tool_name_reverse_map) = converse_tools(&tools, &bedrock)?; + let mut betas: Vec<&str> = Vec::new(); + if bedrock.is_anthropic() { + if tools.iter().any(eager_input_streaming) { + betas.push(FINE_GRAINED_TOOL_STREAMING_BETA); + } + if additional + .get("output_config") + .and_then(|config| config.get("effort")) + .is_some_and(|effort| !effort.is_null()) + && !bedrock.caps.is_adaptive()? + { + betas.push(EFFORT_BETA); + } + let compact_edits: Option<(Map, Vec)> = match additional + .get("context_management") + { + Some(Value::Object(management)) => match management.get("edits") { + Some(Value::Array(edits)) => Some(( + management.clone(), + edits + .iter() + .filter(|edit| { + edit.get("type").and_then(Value::as_str) == Some(COMPACT_EDIT_TYPE) + }) + .cloned() + .collect(), + )), + _ => None, + }, + _ => None, + }; + match compact_edits { + Some((mut management, edits)) if !edits.is_empty() => { + betas.push(COMPACT_BETA); + management.insert("edits".into(), Value::Array(edits)); + additional.insert("context_management".into(), Value::Object(management)); + } + _ => { + additional.remove("context_management"); + } + } + if !betas.is_empty() { + additional.insert("anthropic_beta".into(), json!(betas)); + } + } + additional.remove("cache_control_injection_points"); + + let mut tool_config = None; + if !tool_blocks.is_empty() { + let mut config = Map::from_iter([("tools".to_owned(), Value::Array(tool_blocks))]); + if let Some(choice) = inference.remove("tool_choice") { + config.insert("toolChoice".into(), choice); + if let Some(Value::Object(choice)) = additional.get_mut("tool_choice") { + choice.remove("type"); + } + } + tool_config = Some(Value::Object(config)); + } + let config_blocks: Vec<(&str, Option)> = CONFIG_BLOCKS + .iter() + .map(|name| (*name, inference.remove(*name))) + .collect(); + + let mut body = Map::from_iter([( + "messages".to_owned(), + Value::Array(converse_messages(&messages, &bedrock)?), + )]); + body.insert("inferenceConfig".into(), Value::Object(inference)); + if !additional.is_empty() { + let thinking = additional.contains_key("thinking"); + body.insert( + "additionalModelRequestFields".into(), + Value::Object(additional), + ); + if thinking { + body.insert( + "additionalModelResponseFieldPaths".into(), + json!(["/usage/output_tokens_details"]), + ); + } + } + if !system.is_empty() { + body.insert("system".into(), Value::Array(system)); + } + for (name, value) in config_blocks { + if let Some(value) = value.filter(|value| !value.is_null()) { + body.insert(name.into(), value); + } + } + if let Some(tool_config) = tool_config { + body.insert("toolConfig".into(), tool_config); + } + if let Some(metadata) = request_metadata { + body.insert("requestMetadata".into(), metadata); + } + if let Some(output_config) = output_config { + body.insert("outputConfig".into(), output_config); + } + Ok(ProviderChatRequestData { + body: Value::Object(body), + stream_shape: StreamShape { + json_mode, + tool_name_reverse_map, + response_format_schema, + ..StreamShape::default() + }, + headers: Vec::new(), + }) + } +} + +/// `_transform_thinking_blocks`. +fn thinking_blocks(blocks: &[Value]) -> Vec { + blocks + .iter() + .filter_map(|block| { + if let Some(text) = block.get("reasoningText") { + return Some(ChatCompletionThinkingBlock::Thinking { + thinking: text.get("text").and_then(Value::as_str).map(str::to_owned), + signature: text + .get("signature") + .and_then(Value::as_str) + .map(str::to_owned), + cache_control: None, + index: None, + }); + } + block + .get("redactedContent") + .and_then(Value::as_str) + .map(|data| ChatCompletionThinkingBlock::RedactedThinking { + data: Some(native_redacted_thinking_data(data)), + cache_control: None, + index: None, + }) + }) + .collect() +} + +fn converse_response( + model: &str, + response: ResponseProjection, + shape: &StreamShape, +) -> ChatCompletionsResponse { + let mut content = String::new(); + let mut tool_calls: Vec = Vec::new(); + let mut reasoning_blocks: Option> = None; + let mut citations: Option> = None; + for (index, block) in response + .output + .message + .map(|message| message.content) + .unwrap_or_default() + .into_iter() + .enumerate() + { + let mut extracted_reasoning = None; + if let Some(text) = &block.text { + let (reasoning, answer) = parse_content_for_reasoning(text); + extracted_reasoning = reasoning; + content.push_str(&answer); + } + if let Some(tool) = &block.tool_use { + let name = tool.get("name").and_then(Value::as_str).unwrap_or_default(); + tool_calls.push(ChatCompletionToolCallChunk { + id: tool + .get("toolUseId") + .and_then(Value::as_str) + .map(str::to_owned), + tool_type: "function".into(), + function: ChatCompletionToolCallFunctionChunk { + name: Some( + shape + .tool_name_reverse_map + .get(name) + .cloned() + .unwrap_or_else(|| name.to_owned()), + ), + arguments: python_dumps(tool.get("input").unwrap_or(&Value::Null)), + provider_specific_fields: None, + }, + index: index as i64, + }); + } + if let Some(reasoning) = extracted_reasoning { + reasoning_blocks + .get_or_insert_with(Vec::new) + .push(json!({"reasoningText": {"text": reasoning}})); + } + if let Some(reasoning) = block.reasoning_content { + reasoning_blocks + .get_or_insert_with(Vec::new) + .push(reasoning); + } + if let Some(citation) = block.citations_content { + citations.get_or_insert_with(Vec::new).push(citation); + } + } + let mut provider_fields = Map::new(); + if let Some(blocks) = &reasoning_blocks { + provider_fields.insert( + "reasoningContentBlocks".into(), + Value::Array(blocks.clone()), + ); + } + if let Some(citations) = citations { + provider_fields.insert("citationsContent".into(), Value::Array(citations)); + } + let reasoning_content = reasoning_blocks.as_ref().map(|blocks| { + blocks + .iter() + .filter_map(|block| block.get("reasoningText")?.get("text")?.as_str()) + .collect::() + }); + let thinking = reasoning_blocks.as_deref().map(thinking_blocks); + + // `_filter_json_mode_tools`. + let had_tools = !tool_calls.is_empty(); + let json_indices: Vec = match shape.json_mode { + true => tool_calls + .iter() + .enumerate() + .filter(|(_, call)| call.function.name.as_deref() == Some(RESPONSE_FORMAT_TOOL_NAME)) + .map(|(index, _)| index) + .collect(), + false => Vec::new(), + }; + let filtered: Vec = if json_indices.is_empty() { + tool_calls + } else { + let answer = unwrap_bedrock_properties(&tool_calls[json_indices[0]].function.arguments); + if json_indices.len() == tool_calls.len() { + content = answer; + Vec::new() + } else { + content = match content.is_empty() { + true => answer, + false => format!("{content}{answer}"), + }; + tool_calls + .into_iter() + .enumerate() + .filter(|(index, _)| !json_indices.contains(index)) + .map(|(_, call)| call) + .collect() + } + }; + let mapped = finish_reason_for( + ConverseStopReason::parse(response.stop_reason) + .as_ref() + .map(ConverseStopReason::as_str) + .unwrap_or(""), + ) + .to_string(); + let finish_reason = + match shape.json_mode && filtered.is_empty() && had_tools && mapped == "tool_calls" { + true => "stop".to_owned(), + false => mapped, + }; + + let computed = usage_from_parts( + response.usage.input_tokens, + response.usage.output_tokens, + response.usage.cache_read_input_tokens, + response.usage.cache_write_input_tokens, + ); + // Converse reports `totalTokens` and Python passes it straight through. Python does a + // bare `usage["totalTokens"]` lookup, so a body without the key raises there; the + // computed total stands in rather than failing the call. + let usage = ChatCompletionsUsage { + total_tokens: response.usage.total_tokens.unwrap_or(computed.total_tokens), + ..computed + }; + ChatCompletionsResponse { + created: unix_now(), + // Converse echoes no model id, so Python reports the requested one. + model: model.to_string(), + choices: vec![ChatCompletionsChoice { + index: 0, + message: ChatCompletionsChoiceMessage { + role: "assistant".to_string(), + // Converse assigns the joined string unconditionally, so an empty answer + // is `""` here and not `None` as it is on Anthropic. + content: Some(content), + tool_calls: (!filtered.is_empty()).then_some(filtered), + reasoning_content, + thinking_blocks: thinking, + provider_specific_fields: (!provider_fields.is_empty()).then_some(provider_fields), + }, + finish_reason, + }], + usage, + } +} + impl BaseConfig for AmazonConverseConfig { fn secret_names(&self) -> Vec<&'static str> { litellm_auth::AwsParams::secret_names() @@ -105,10 +674,6 @@ impl BaseConfig for AmazonConverseConfig { .collect() } - fn supported_openai_param_mappings(&self) -> &'static [(&'static str, &'static str)] { - SUPPORTED_PARAMS - } - fn get_complete_url( &self, api_base: Option<&str>, @@ -142,20 +707,19 @@ impl BaseConfig for AmazonConverseConfig { fn transform_request( &self, - _model: &str, + model: &str, messages: Vec, optional_params: Map, + context: &ChatRequestContext<'_>, ) -> Result { - Ok(ProviderChatRequestData { - body: converse_body(&build_conversation(&messages), &optional_params), - stream_shape: StreamShape::default(), - }) + self.build_request(model, messages, optional_params, context) } fn transform_response( &self, model: &str, response: ProviderChatResponseData, + shape: &StreamShape, ) -> Result { let body = response.body; if !body.is_object() { @@ -170,55 +734,18 @@ impl BaseConfig for AmazonConverseConfig { )); } } - let response: ConverseResponse = serde_json::from_value(body).map_err(|error| { + let response: ResponseProjection = serde_json::from_value(body).map_err(|error| { Error::InvalidResponse(format!("invalid Converse response: {error}").into()) })?; - // The route declines tool requests, so anything other than a text block - // is something this path never asked for. Decline; the host falls back. - if response.message_content_is_non_text() { - return Err(Error::Unsupported("non-text response content block")); - } - let text = response.content_text(); - let computed = usage_from_parts( - response.usage.input_tokens, - response.usage.output_tokens, - response.usage.cache_read_input_tokens, - response.usage.cache_write_input_tokens, - ); - // Converse reports `totalTokens` and Python passes it straight through, - // where Anthropic has no such field and Python adds the two counts - // instead, so only this provider overrides the computed total. Python - // does a bare `usage["totalTokens"]` lookup, so a body without the key - // raises there rather than reporting a zero; fall back to the computed - // total, which is the closest thing to that without failing the call. - let usage = ChatCompletionsUsage { - total_tokens: response.usage.total_tokens.unwrap_or(computed.total_tokens), - ..computed - }; + Ok(converse_response(model, response, shape)) + } - Ok(ChatCompletionsResponse { - created: unix_now(), - // Converse echoes no model id, so Python reports the requested one. - model: model.to_string(), - choices: vec![ChatCompletionsChoice { - index: 0, - message: ChatCompletionsChoiceMessage { - role: "assistant".to_string(), - // Converse assigns the joined string unconditionally, so an - // empty response is `""` here and not `None` as it is on - // Anthropic. A caller calling `.strip()` on it would break - // on this path alone. - content: Some(text), - }, - finish_reason: finish_reason_for( - ConverseStopReason::parse(response.stop_reason) - .as_ref() - .map(ConverseStopReason::as_str) - .unwrap_or(""), - ) - .to_string(), - }], - usage, + /// Python's `BedrockImageProcessor` downloads every image or file URL a message names. + fn remote_media(&self, messages: &[ChatMessage]) -> Vec { + remote_urls(messages, &["user", "assistant", "tool"], true, |url| { + data_url(url).is_none() + && !url.contains("base64") + && (url.contains("http://") || url.contains("https://")) }) } @@ -282,100 +809,30 @@ impl BaseConfig for AmazonConverseConfig { CONFIG_PARAMS } + /// Every request is translated; a param or message Python refuses is refused here too. fn unsupported_reason( &self, - messages: &[ChatMessage], - optional_params: &Map, + _messages: &[ChatMessage], + _optional_params: &Map, ) -> Option { - unsupported_param( - self.supported_openai_param_mappings(), - CONFIG_PARAMS, - optional_params, - ) - .or_else(|| messages.iter().find_map(unsupported_message)) - // Python's Converse translation drops blank text blocks instead of - // substituting the placeholder the shared conversation builder - // applies, so decline blank text rather than diverge. - .or_else(|| { - messages - .iter() - .any(has_blank_text) - .then_some(Unsupported("blank message text")) - }) - // Converse has no assistant prefill: Python inserts a continue turn - // when a conversation opens or closes on an assistant message, and - // only under `litellm.modify_params`, which the core cannot see. - // Declining both ends also keeps the shared builder's final - // assistant right-strip (an Anthropic rule) unreachable here. - .or_else(|| { - let conversation = build_conversation(messages); - let ends_on_assistant = conversation - .turns - .last() - .is_some_and(|turn| turn.role == TurnRole::Assistant); - (!conversation.opens_on_user_turn() || ends_on_assistant).then_some(Unsupported( - "conversation does not run user turn to user turn", - )) - }) - } -} - -fn converse_body(conversation: &Conversation, optional_params: &Map) -> Value { - let messages: Vec = conversation - .turns - .iter() - .map(|turn| { - json!({ - "role": <&'static str>::from(turn.role), - "content": turn.texts.iter().map(|text| json!({"text": text})).collect::>(), - }) - }) - .collect(); - - let inference_config = Map::from_iter(SUPPORTED_PARAMS.iter().filter_map(|(_, name)| { - optional_params - .get(*name) - .map(|value| ((*name).to_string(), value.clone())) - })); - - let system: Vec = conversation - .system - .iter() - .map(|text| json!({"text": text})) - .collect(); - - Value::Object(Map::from_iter( - [ - ( - "inferenceConfig".to_string(), - Value::Object(inference_config), - ), - ("messages".to_string(), json!(messages)), - ] - .into_iter() - .chain((!system.is_empty()).then(|| ("system".to_string(), json!(system)))), - )) -} - -fn has_blank_text(message: &ChatMessage) -> bool { - match &message.content { - None => false, - Some(ChatMessageContent::Text(text)) => text.trim().is_empty(), - Some(ChatMessageContent::Parts(parts)) => parts.iter().any(|part| { - part.get("text") - .and_then(Value::as_str) - .is_none_or(|text| text.trim().is_empty()) - }), + None } } #[cfg(test)] mod tests { + use std::collections::HashMap as StdHashMap; + use crate::{ Error, base_llm::{ auth::AuthScheme, - chat::transformation::{BaseConfig, ProviderChatResponseData, Unsupported}, + chat::{ + params::ChatRequestContext, + streaming::StreamShape, + transformation::{BaseConfig, ProviderChatResponseData}, + }, + model_info::ModelInfo, }, bedrock::chat::converse_transformation::BEDROCK_CHAT_COMPLETIONS_CONFIG, }; @@ -384,6 +841,27 @@ mod tests { use rstest::rstest; use serde_json::{Map, Value, json}; + struct CostMap(StdHashMap>); + + impl ModelInfo for CostMap { + fn entry(&self, model: &str) -> Option>> { + self.0.get(model).map(std::borrow::Cow::Borrowed) + } + fn declares(&self, key: &str) -> bool { + self.0.values().any(|entry| entry.contains_key(key)) + } + } + + fn cost_map() -> CostMap { + CostMap(StdHashMap::from([( + "anthropic.claude-sonnet-4-5-v1:0".to_owned(), + json!({"supports_tool_choice": true, "supports_reasoning": true, "supports_prompt_caching": true}) + .as_object() + .cloned() + .unwrap(), + )])) + } + fn messages(value: Value) -> Vec { serde_json::from_value(value).expect("valid messages") } @@ -395,207 +873,204 @@ mod tests { } } - fn transform(msgs: Value, opts: Value) -> Value { - BEDROCK_CHAT_COMPLETIONS_CONFIG - .transform_request( - "anthropic.claude-sonnet-4-5-v1:0", - messages(msgs), - params(opts), - ) - .expect("request transforms") - .body - } - - fn transform_response(body: Value) -> Result { - BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( + fn request( + msgs: Value, + opts: Value, + drop: bool, + ) -> Result { + let info = cost_map(); + BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_request( "anthropic.claude-sonnet-4-5-v1:0", - ProviderChatResponseData { body }, + messages(msgs), + params(opts), + &ChatRequestContext { + drop_params: Some(drop), + model_info: Some(&info), + }, ) } - fn reason(msgs: Value, opts: Value) -> Option { - BEDROCK_CHAT_COMPLETIONS_CONFIG.unsupported_reason(&messages(msgs), ¶ms(opts)) + fn transform(msgs: Value, opts: Value) -> Value { + request(msgs, opts, false).expect("request transforms").body } - #[test] + fn transform_response(body: Value) -> Result { + transform_response_with(body, &StreamShape::default()) + } + + fn transform_response_with( + body: Value, + shape: &StreamShape, + ) -> Result { + BEDROCK_CHAT_COMPLETIONS_CONFIG.transform_response( + "anthropic.claude-sonnet-4-5-v1:0", + ProviderChatResponseData { body }, + shape, + ) + } + + #[rstest] fn builds_the_converse_body_python_builds() { let body = transform( json!([ {"role": "system", "content": "be terse"}, {"role": "user", "content": "hi"} ]), - json!({"maxTokens": 128, "temperature": 0.2}), + json!({"max_tokens": 128, "temperature": 0.2, "top_p": 0.9, "stop": "END"}), ); assert_eq!( body, json!({ - "inferenceConfig": {"maxTokens": 128, "temperature": 0.2}, "messages": [{"role": "user", "content": [{"text": "hi"}]}], + "inferenceConfig": {"maxTokens": 128, "temperature": 0.2, "topP": 0.9, "stopSequences": ["END"]}, "system": [{"text": "be terse"}] }) ); } - #[test] - fn always_emits_inference_config_even_when_empty() { - let body = transform(json!([{"role": "user", "content": "hi"}]), json!({})); - assert_eq!(body["inferenceConfig"], json!({})); - assert!(body.get("system").is_none()); - } - - #[test] - fn places_only_inference_params_in_inference_config() { + #[rstest] + fn tools_and_tool_choice_become_a_tool_config() { let body = transform( - json!([{"role": "user", "content": "hi"}]), + json!([{"role": "user", "content": "weather?"}]), json!({ - "maxTokens": 64, - "temperature": 0.1, - "topP": 0.9, - "stopSequences": ["STOP"] + "max_tokens": 10, + "tools": [{"type": "function", "function": {"name": "get_weather", "description": "d", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}}], + "tool_choice": "required" }), ); assert_eq!( - body["inferenceConfig"], - json!({"maxTokens": 64, "temperature": 0.1, "topP": 0.9, "stopSequences": ["STOP"]}) + body["toolConfig"], + json!({"tools": [{"toolSpec": {"inputSchema": {"json": {"type": "object", "properties": {"city": {"type": "string"}}, "required": []}}, "name": "get_weather", "description": "d"}}], + "toolChoice": {"any": {}}}) ); - assert!(body.get("additionalModelRequestFields").is_none()); } - #[test] - fn merges_consecutive_user_turns_into_one_message() { + #[rstest] + fn tool_choice_none_is_refused_or_dropped() { + let msgs = json!([{"role": "user", "content": "hi"}]); + let error = request(msgs.clone(), json!({"tool_choice": "none"}), false) + .map(|_| ()) + .unwrap_err(); + assert_eq!( + error.to_string(), + "invalid request: Bedrock doesn't support tool_choice=none. To drop it from the call, set `litellm.drop_params = True." + ); + let body = request(msgs, json!({"tool_choice": "none"}), true) + .unwrap() + .body; + assert!(body.get("toolConfig").is_none()); + } + + #[rstest] + fn unsupported_params_are_refused_or_dropped() { + let msgs = json!([{"role": "user", "content": "hi"}]); + assert!(request(msgs.clone(), json!({"seed": 3}), false).is_err()); + let body = request(msgs, json!({"seed": 3}), true).unwrap().body; + assert!(body.get("additionalModelRequestFields").is_none()); + assert_eq!(body["inferenceConfig"], json!({})); + } + + #[rstest] + fn reasoning_effort_becomes_thinking_in_additional_fields() { + let body = transform( + json!([{"role": "user", "content": "hi"}]), + json!({"reasoning_effort": "low"}), + ); + assert_eq!( + body["additionalModelRequestFields"], + json!({"thinking": {"type": "enabled", "budget_tokens": 1024}}) + ); + assert_eq!( + body["additionalModelResponseFieldPaths"], + json!(["/usage/output_tokens_details"]) + ); + assert_eq!(body["inferenceConfig"]["maxTokens"], json!(1024 + 4096)); + } + + #[rstest] + fn response_format_becomes_a_json_tool() { + let data = request( + json!([{"role": "user", "content": "hi"}]), + json!({"response_format": {"type": "json_schema", "json_schema": {"name": "x", + "schema": {"type": "object", "properties": {"a": {"type": "string"}}}}}}), + false, + ) + .unwrap(); + assert_eq!( + data.body["toolConfig"]["toolChoice"], + json!({"tool": {"name": "json_tool_call"}}) + ); + assert!(data.stream_shape.json_mode); + let response = transform_response_with( + json!({"output": {"message": {"content": [ + {"toolUse": {"toolUseId": "t", "name": "json_tool_call", "input": {"properties": {"a": "b"}}}} + ]}}, "stopReason": "tool_use", "usage": {"inputTokens": 1, "outputTokens": 1, "totalTokens": 2}}), + &data.stream_shape, + ) + .unwrap(); + assert_eq!( + response.choices[0].message.content.as_deref(), + Some("{\"a\": \"b\"}") + ); + assert_eq!(response.choices[0].finish_reason, "stop"); + } + + #[rstest] + fn orphaned_tool_calls_are_sent_as_text() { let body = transform( json!([ - {"role": "user", "content": "one"}, - {"role": "user", "content": [{"type": "text", "text": "two"}]}, - {"role": "assistant", "content": "ack"}, - {"role": "user", "content": "three"} + {"role": "user", "content": "hi"}, + {"role": "assistant", "tool_calls": [{"id": "c", "type": "function", "function": {"name": "f", "arguments": "{}"}}]}, + {"role": "tool", "tool_call_id": "c", "content": "ok"} ]), json!({}), ); assert_eq!( - body["messages"], - json!([ - {"role": "user", "content": [{"text": "one"}, {"text": "two"}]}, - {"role": "assistant", "content": [{"text": "ack"}]}, - {"role": "user", "content": [{"text": "three"}]} - ]) - ); - } - - #[test] - fn declines_streaming() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"stream": true}) - ), - Some(Unsupported("streaming")) - ); - } - - #[test] - fn declines_top_k_because_python_routes_it_by_base_model() { - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - json!({"topK": 40}) - ), - Some(Unsupported("unrecognized request parameter")) + body["messages"][1]["content"][0]["text"], + "[tool call c: f({})]" ); } #[rstest] - #[case::tools(json!({"tools": []}))] - #[case::tool_choice(json!({"tool_choice": {"auto": {}}}))] - #[case::thinking(json!({"thinking": {"type": "enabled"}}))] - #[case::request_metadata(json!({"requestMetadata": {"k": "v"}}))] - #[case::output_config(json!({"outputConfig": {}}))] - #[case::parallel_tool_use_config(json!({"_parallel_tool_use_config": {}}))] - fn declines_tools_and_other_params_outside_the_allowlist(#[case] param: Value) { + fn tool_use_and_reasoning_come_back_as_openai_fields() { + let shape = StreamShape { + tool_name_reverse_map: StdHashMap::from([("get_weather".into(), "get.weather".into())]), + ..StreamShape::default() + }; + let response = transform_response_with( + json!({"output": {"message": {"content": [ + {"reasoningContent": {"reasoningText": {"text": "hmm", "signature": "sig"}}}, + {"text": "checking"}, + {"toolUse": {"toolUseId": "t1", "name": "get_weather", "input": {"city": "Paris"}}} + ]}}, "stopReason": "tool_use", "usage": {"inputTokens": 1, "outputTokens": 1}}), + &shape, + ) + .unwrap(); + let message = serde_json::to_value(&response.choices[0].message).unwrap(); assert_eq!( - reason(json!([{"role": "user", "content": "hi"}]), param.clone()), - Some(Unsupported("unrecognized request parameter")), - "expected {param} to decline" + message["tool_calls"], + json!([{"id": "t1", "type": "function", "function": {"name": "get.weather", "arguments": "{\"city\": \"Paris\"}"}, "index": 2}]) ); + assert_eq!(message["reasoning_content"], "hmm"); + assert_eq!( + message["thinking_blocks"], + json!([{"type": "thinking", "thinking": "hmm", "signature": "sig"}]) + ); + assert_eq!(response.choices[0].finish_reason, "tool_calls"); } #[rstest] - #[case::empty_string(json!(""))] - #[case::whitespace_string(json!(" "))] - #[case::whitespace_text_block(json!([{"type": "text", "text": " "}]))] - fn declines_blank_text_rather_than_substituting_the_anthropic_placeholder( - #[case] content: Value, + #[case("ab", Some("a"), "b")] + #[case("plain", None, "plain")] + fn leading_think_tags_are_reasoning( + #[case] text: &str, + #[case] reasoning: Option<&str>, + #[case] answer: &str, ) { - assert_eq!( - reason( - json!([{"role": "user", "content": content}, {"role": "user", "content": "hi"}]), - json!({}) - ), - Some(Unsupported("blank message text")), - "expected blank content {content} to decline" - ); - } - - #[test] - fn declines_a_message_whose_content_list_is_empty() { - // The blank-text check scans parts, so an empty list clears it; Converse - // rejects an empty `content` array, which is a decline the core owes the - // host before the call rather than an error after it. - assert_eq!( - reason(json!([{"role": "user", "content": []}]), json!({})), - Some(Unsupported("message without content")) - ); - assert_eq!( - reason( - json!([{"role": "user", "content": [{"type": "text", "text": "hi"}]}]), - json!({}) - ), - None - ); - } - - #[test] - fn declines_a_conversation_that_opens_or_closes_on_an_assistant_turn() { - assert_eq!( - reason( - json!([ - {"role": "assistant", "content": "prefill"}, - {"role": "user", "content": "hi"} - ]), - json!({}) - ), - Some(Unsupported( - "conversation does not run user turn to user turn" - )) - ); - assert_eq!( - reason( - json!([ - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "prefill"} - ]), - json!({}) - ), - Some(Unsupported( - "conversation does not run user turn to user turn" - )) - ); - } - - #[test] - fn accepts_a_user_to_user_text_conversation() { - assert_eq!( - reason( - json!([ - {"role": "system", "content": "be terse"}, - {"role": "user", "content": "hi"}, - {"role": "assistant", "content": "hello"}, - {"role": "user", "content": "again"} - ]), - json!({"maxTokens": 16}) - ), - None - ); + let (found, rest) = super::parse_content_for_reasoning(text); + assert_eq!(found.as_deref(), reasoning); + assert_eq!(rest, answer); } #[test] @@ -822,24 +1297,6 @@ mod tests { assert_eq!(response.usage.total_tokens, 14); } - #[test] - fn declines_a_cache_control_message_so_widening_the_gate_is_a_red_test() { - // Converse only reports cache token counts when the request carries a - // cachePoint block, which is why the provider total and the computed one - // cannot disagree today. This is the tripwire: whoever widens the gate to - // admit prompt caching has to come back and re-check the usage mapping - // rather than discovering a silent number change in production. - assert_eq!( - reason( - json!([{"role": "user", "content": [ - {"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}} - ]}]), - json!({}) - ), - Some(Unsupported("non-text message content")) - ); - } - #[test] fn folds_converse_cache_tokens_into_prompt_tokens() { let response = transform_response(json!({ @@ -862,19 +1319,6 @@ mod tests { assert_eq!(response.usage.prompt_tokens_details.text_tokens, 10); } - #[test] - fn declines_a_response_carrying_a_tool_use_block() { - let err = transform_response(json!({ - "output": {"message": {"content": [ - {"toolUse": {"toolUseId": "t1", "name": "f", "input": {}}} - ]}}, - "stopReason": "tool_use", - "usage": {"inputTokens": 1, "outputTokens": 1} - })) - .expect_err("tool use block"); - assert_eq!(err, Error::Unsupported("non-text response content block")); - } - #[test] fn errors_on_a_response_missing_required_fields() { assert_eq!( @@ -908,7 +1352,7 @@ mod tests { #[test] fn accepts_aws_call_configuration_without_serializing_it() { let call_config = json!({ - "maxTokens": 16, + "max_tokens": 16, "aws_access_key_id": "AKIA", "aws_secret_access_key": "secret", "aws_session_token": "token", @@ -921,19 +1365,12 @@ mod tests { "aws_external_id": "ext", "aws_bedrock_runtime_endpoint": "https://vpce.internal" }); - assert_eq!( - reason( - json!([{"role": "user", "content": "hi"}]), - call_config.clone() - ), - None - ); let body = transform(json!([{"role": "user", "content": "hi"}]), call_config); assert_eq!( body, json!({ - "inferenceConfig": {"maxTokens": 16}, - "messages": [{"role": "user", "content": [{"text": "hi"}]}] + "messages": [{"role": "user", "content": [{"text": "hi"}]}], + "inferenceConfig": {"maxTokens": 16} }), "aws call configuration must not reach the Converse body" ); diff --git a/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs b/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs index d5ae8d6a323..3ec0caed6ae 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/invoke_handler.rs @@ -23,9 +23,13 @@ use crate::{ anthropic::chat::handler::ModelResponseIterator, base_llm::{ base_model_iterator::StreamTransformer, - chat::streaming::{ChatStream, StreamShape}, + chat::{ + params::RESPONSE_FORMAT_TOOL_NAME, + streaming::{ChatStream, StreamShape}, + }, messages::streaming::{ByteStream, EventStream}, }, + bedrock::chat::converse_transformation::unwrap_bedrock_properties, }; #[derive(Deserialize)] @@ -200,7 +204,13 @@ pub fn converse_event_stream(bytes: ByteStream) -> BoxStream<'static, Result, + /// The block in progress is the `json_tool_call` tool a `response_format` became. + in_response_format_tool: bool, + /// Some `json_tool_call` block has been streamed back as text. + converted_response_format_tool: bool, } #[derive(Default)] @@ -233,6 +247,8 @@ impl ConverseStreamIterator { thinking_ran: false, thinking_block_index: -1, thinking_content_block_index: None, + in_response_format_tool: false, + converted_response_format_tool: false, } } @@ -295,7 +311,15 @@ impl ConverseStreamIterator { fn start(&mut self, start: &Value, index: u64) -> ConverseParsed { let mut parsed = ConverseParsed::default(); self.content_blocks.clear(); + self.in_response_format_tool = false; if let Some(tool) = start.get("toolUse").filter(|tool| !tool.is_null()) { + if self.shape.json_mode + && tool.get("name").and_then(Value::as_str) == Some(RESPONSE_FORMAT_TOOL_NAME) + { + self.in_response_format_tool = true; + self.converted_response_format_tool = true; + return parsed; + } self.tool_calls_index = Some(self.tool_calls_index.map_or(0, |index| index + 1)); let name = tool.get("name").and_then(Value::as_str).map(|name| { self.shape @@ -329,6 +353,9 @@ impl ConverseStreamIterator { .unwrap_or_default() .to_owned(); self.content_blocks.push(input.clone()); + if self.in_response_format_tool { + return parsed; + } parsed.tool_use = Some(self.tool_call(None, None, input, index)); } else if let Some(reasoning) = delta.get("reasoningContent") { parsed @@ -359,6 +386,13 @@ impl ConverseStreamIterator { self.start(start, index) } else if let Some(delta) = data.get("delta") { self.delta(delta, index) + } else if data.get("contentBlockIndex").is_some() && self.in_response_format_tool { + self.in_response_format_tool = false; + let answer = std::mem::take(&mut self.content_blocks).concat(); + ConverseParsed { + text: unwrap_bedrock_properties(if answer.is_empty() { "{}" } else { &answer }), + ..ConverseParsed::default() + } } else if data.get("contentBlockIndex").is_some() { // A block's stop: a tool that streamed only empty fragments closes with `{}`. let empty = @@ -368,10 +402,15 @@ impl ConverseStreamIterator { ..ConverseParsed::default() } } else if let Some(reason) = data.get("stopReason") { + let mut finish = finish_reason_for(reason.as_str().unwrap_or("stop")); + if finish == "tool_calls" + && self.converted_response_format_tool + && self.tool_calls_index.is_none() + { + finish = "stop"; + } ConverseParsed { - finish_reason: Some( - finish_reason_for(reason.as_str().unwrap_or("stop")).to_owned(), - ), + finish_reason: Some(finish.to_owned()), ..ConverseParsed::default() } } else if let Some(usage) = data.get("usage") { @@ -735,6 +774,39 @@ mod tests { assert_eq!(chunks[7]["choices"][0]["finish_reason"], "tool_calls"); } + #[test] + fn a_response_format_tool_streams_back_as_text() { + let mut iterator = ConverseStreamIterator::new(StreamShape { + json_mode: true, + ..StreamShape::default() + }); + let chunks: Vec = serde_json::json!([ + {"contentBlockIndex": 0, "start": {"toolUse": {"toolUseId": "t1", "name": "json_tool_call"}}}, + {"contentBlockIndex": 0, "delta": {"toolUse": {"input": "{\"properties\": {\"a\""}}}, + {"contentBlockIndex": 0, "delta": {"toolUse": {"input": ": 1}}"}}}, + {"contentBlockIndex": 0}, + {"stopReason": "tool_use"} + ]) + .as_array() + .unwrap() + .iter() + .flat_map(|event| iterator.transform(event.clone()).unwrap()) + .map(|chunk| serde_json::to_value(chunk).unwrap()) + .collect(); + for chunk in &chunks { + assert_eq!( + chunk["choices"][0]["delta"]["tool_calls"], + serde_json::Value::Null + ); + } + let content: String = chunks + .iter() + .filter_map(|chunk| chunk["choices"][0]["delta"]["content"].as_str()) + .collect(); + assert_eq!(content, "{\"a\": 1}"); + assert_eq!(chunks[4]["choices"][0]["finish_reason"], "stop"); + } + #[test] fn converse_reasoning_streams_as_thinking_blocks() { let chunks = converse_chunks(serde_json::json!([ diff --git a/litellm-rust/crates/llms/src/bedrock/chat/mod.rs b/litellm-rust/crates/llms/src/bedrock/chat/mod.rs index a46514aa697..e35bd6d88e7 100644 --- a/litellm-rust/crates/llms/src/bedrock/chat/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock/chat/mod.rs @@ -1,2 +1,4 @@ +pub mod converse_messages; +pub mod converse_params; pub mod converse_transformation; pub mod invoke_handler; diff --git a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs index 5ca084cd517..d8d580f3074 100644 --- a/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs +++ b/litellm-rust/crates/llms/src/openai_like/chat/transformation.rs @@ -16,9 +16,13 @@ use crate::{ Error, base_llm::{ auth::AuthScheme, - chat::transformation::{ - BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, - ValidatedEnvironment, + chat::{ + params::ChatRequestContext, + streaming::StreamShape, + transformation::{ + BaseConfig, Headers, ProviderChatRequestData, ProviderChatResponseData, + ValidatedEnvironment, + }, }, }, openai_like::common_utils::{complete_openai_like_url, openai_compatible_provider_info}, @@ -90,6 +94,7 @@ impl BaseConfig for OpenAILikeChatConfig { model: &str, messages: Vec, optional_params: Map, + _context: &ChatRequestContext<'_>, ) -> Result { let mut params = Map::from_iter( optional_params @@ -113,6 +118,7 @@ impl BaseConfig for OpenAILikeChatConfig { Ok(ProviderChatRequestData { body: Value::Object(body), stream_shape: Default::default(), + headers: Vec::new(), }) } @@ -120,6 +126,7 @@ impl BaseConfig for OpenAILikeChatConfig { &self, model: &str, response: ProviderChatResponseData, + _shape: &StreamShape, ) -> Result { let mut body = response.body; sanitize_usage(&mut body); @@ -266,6 +273,7 @@ fn normalize_choice(position: usize, choice: &Value) -> Result Value { OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .transform_request(model, messages(msgs), params(opts)) + .transform_request(model, messages(msgs), params(opts), &Default::default()) .expect("request transforms") .body } fn transform_response(body: Value) -> Result { - OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG - .transform_response("some-model", ProviderChatResponseData { body }) + OPENAI_LIKE_CHAT_COMPLETIONS_CONFIG.transform_response( + "some-model", + ProviderChatResponseData { body }, + &Default::default(), + ) } fn reason(msgs: Value, opts: Value) -> Option { diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs index 5ad9f8df8b8..dd2c079082c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/host.rs @@ -1,5 +1,6 @@ -use std::convert::Infallible; +use std::{convert::Infallible, sync::Arc}; +use super::model_cost::PythonModelCost; use crate::routes::inference::InferenceHost; use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned}; use litellm_inference_chat::{Error, route::ChatCompletions, types::ChatCompletionsCall}; @@ -26,6 +27,10 @@ pub(super) fn project( custom_llm_provider: call.options.custom_llm_provider, extra_headers: call.options.extra_headers, timeout: call.options.timeout, + // Python's own settings decide what a drop would do, so a param that needs one goes + // back to Python. Capabilities come from the cost map Python itself reads. + drop_params: None, + model_info: Some(Arc::new(PythonModelCost::current(py)?)), }) } diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs index d78d141f0f2..ce319189a60 100644 --- a/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/mod.rs @@ -1,4 +1,5 @@ mod host; +mod model_cost; use std::sync::Arc; diff --git a/litellm-rust/crates/python-bridge/src/routes/chat_completions/model_cost.rs b/litellm-rust/crates/python-bridge/src/routes/chat_completions/model_cost.rs new file mode 100644 index 00000000000..09a7e64f101 --- /dev/null +++ b/litellm-rust/crates/python-bridge/src/routes/chat_completions/model_cost.rs @@ -0,0 +1,88 @@ +//! Python's `litellm.model_cost` as the cost map chat translation reads, so a capability +//! lookup sees the map Python itself would, models registered at runtime included. + +use std::{borrow::Cow, collections::HashSet, sync::OnceLock}; + +use litellm_host_python::from_py; +use litellm_llms::base_llm::model_info::ModelInfo; +use pyo3::{prelude::*, types::PyDict}; +use serde_json::{Map, Value}; + +pub(super) struct PythonModelCost { + map: Py, + declared: OnceLock>, +} + +impl PythonModelCost { + /// The dict `litellm.model_cost` holds now. A map Python replaces later is not followed, + /// as a call already under way would not follow it either. + pub(super) fn current(py: Python<'_>) -> PyResult { + let map = py + .import("litellm")? + .getattr("model_cost")? + .cast_into::()? + .unbind(); + Ok(Self { + map, + declared: OnceLock::new(), + }) + } +} + +impl ModelInfo for PythonModelCost { + fn entry(&self, model: &str) -> Option>> { + Python::attach(|py| { + let entry = self.map.bind(py).get_item(model).ok()??; + from_py::>(&entry).ok().map(Cow::Owned) + }) + } + + fn declares(&self, key: &str) -> bool { + self.declared + .get_or_init(|| { + Python::attach(|py| { + self.map + .bind(py) + .values() + .iter() + .filter_map(|entry| entry.cast_into::().ok()) + .flat_map(|entry| { + entry + .keys() + .iter() + .filter_map(|key| key.extract::().ok()) + .collect::>() + }) + .collect() + }) + }) + .contains(key) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn reads_the_entry_python_holds_and_the_keys_any_entry_sets() { + Python::initialize(); + Python::attach(|py| { + let map = PyDict::new(py); + let entry = PyDict::new(py); + entry.set_item("supports_reasoning", true).unwrap(); + entry.set_item("max_output_tokens", 64000).unwrap(); + map.set_item("claude-x", entry).unwrap(); + let cost = PythonModelCost { + map: map.unbind(), + declared: OnceLock::new(), + }; + let found = cost.entry("claude-x").unwrap(); + assert_eq!(found.get("supports_reasoning"), Some(&Value::Bool(true))); + assert_eq!(found.get("max_output_tokens"), Some(&Value::from(64000))); + assert!(cost.entry("unknown").is_none()); + assert!(cost.declares("max_output_tokens")); + assert!(!cost.declares("default_reasoning_effort")); + }); + } +} diff --git a/tests/test_litellm_rust/chat_completions/test_callbacks.py b/tests/test_litellm_rust/chat_completions/test_callbacks.py index 6743f9adbf0..ba5461bd24f 100644 --- a/tests/test_litellm_rust/chat_completions/test_callbacks.py +++ b/tests/test_litellm_rust/chat_completions/test_callbacks.py @@ -22,13 +22,13 @@ _OBJECT: Final = TypeAdapter(dict[str, object]) async def test_returns_a_model_response_and_logs_success_once(server: RecordingServer, asynchronous: bool) -> None: recorder: Final = RecordingLogger() - result: Final = await call(server, asynchronous=asynchronous, callbacks=[recorder], temperature=0.25) + result: Final = await call(server, asynchronous=asynchronous, callbacks=[recorder], temperature=1) assert_served_natively(server) sent: Final = server.requests[0] body: Final = _OBJECT.validate_python(sent.body) assert sent.path == "/v1/messages" - assert body["temperature"] == 0.25 + assert body["temperature"] == 1 assert request_body(_OBJECT.validate_python(recorder.wait_for("log_pre_api_call")[0].kwargs)) == body assert isinstance(result, ModelResponse) assert result.choices[0].message.content == MESSAGES_TEXT diff --git a/tests/test_litellm_rust/chat_completions/test_requests.py b/tests/test_litellm_rust/chat_completions/test_requests.py index c8800501cd4..41aa9b33932 100644 --- a/tests/test_litellm_rust/chat_completions/test_requests.py +++ b/tests/test_litellm_rust/chat_completions/test_requests.py @@ -18,7 +18,8 @@ _OBJECT: Final = TypeAdapter(dict[str, object]) @pytest.mark.asyncio @pytest.mark.parametrize("asynchronous", (False, True)) async def test_positional_parameters_reach_the_wire(server: RecordingServer, asynchronous: bool) -> None: - temperature: Final = 0.35 + # The model takes no sampling params but temperature=1, as Python itself rules. + temperature: Final = 1.0 arguments: Final = ( (MESSAGES_MODEL, list(MESSAGES), None, None, 12.0, temperature) if asynchronous