From 0423ac1164c4c04f2b525cc1c90be28d4baaa404 Mon Sep 17 00:00:00 2001 From: narmstrong-berry Date: Thu, 15 Oct 2026 03:25:58 +0000 Subject: [PATCH] feat(rust): serve Bedrock Mantle responses natively The Responses route now resolves bedrock_mantle. The Mantle config sends the OpenAI Responses payload to the /openai/v1 or /v1 path the model cost map picks, with a bearer token or AWS SigV4, and re-pins a standard Mantle host to the resolved region. It keeps only the tool types Mantle accepts, hoists additional_tools items and rewrites Codex history items. Chat-only models, emulated file_search and service_tier or reasoning.summary values Mantle rejects stay on Python. --- litellm-rust/Cargo.lock | 1 + litellm-rust/crates/auth-aws/src/aws.rs | 2 +- .../crates/gateway-inference/src/responses.rs | 1 + .../crates/inference-responses/Cargo.toml | 1 + .../crates/inference-responses/src/prepare.rs | 73 ++- .../crates/inference-responses/src/types.rs | 5 +- .../inference-responses/tests/caching.rs | 1 + .../tests/live/openrouter.rs | 1 + .../inference-responses/tests/openrouter.rs | 1 + .../inference-responses/tests/responses.rs | 1 + litellm-rust/crates/llms/AGENTS.md | 2 +- .../base_llm/responses/additional_tools.rs | 72 +++ .../src/base_llm/responses/codex_compat.rs | 152 ++++++ .../crates/llms/src/base_llm/responses/mod.rs | 2 + .../crates/llms/src/bedrock_mantle/mod.rs | 1 + .../src/bedrock_mantle/responses/AGENTS.md | 12 + .../llms/src/bedrock_mantle/responses/mod.rs | 1 + .../responses/transformation.rs | 504 ++++++++++++++++++ .../llms/src/xai/responses/transformation.rs | 14 +- .../src/routes/responses/host.rs | 39 +- .../python-bridge/src/routes/responses/mod.rs | 4 +- .../crates/python-compat/src/truthy.rs | 12 + litellm/rust_bridge/host/routes/responses.py | 44 +- .../sdk/responses/test_native_parity.py | 49 +- .../rust_bridge/host/routes/test_responses.py | 79 +++ 25 files changed, 1030 insertions(+), 44 deletions(-) create mode 100644 litellm-rust/crates/llms/src/base_llm/responses/additional_tools.rs create mode 100644 litellm-rust/crates/llms/src/base_llm/responses/codex_compat.rs create mode 100644 litellm-rust/crates/llms/src/bedrock_mantle/responses/AGENTS.md create mode 100644 litellm-rust/crates/llms/src/bedrock_mantle/responses/mod.rs create mode 100644 litellm-rust/crates/llms/src/bedrock_mantle/responses/transformation.rs diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index a736315ed08..ef72eec8a48 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4098,6 +4098,7 @@ dependencies = [ "litellm-inference-testing", "litellm-llms", "litellm-llms-types", + "litellm-router-types", "litellm-secrets", "litellm-tracing", "reqwest 0.12.28", diff --git a/litellm-rust/crates/auth-aws/src/aws.rs b/litellm-rust/crates/auth-aws/src/aws.rs index 8fdbba21711..42c848185de 100644 --- a/litellm-rust/crates/auth-aws/src/aws.rs +++ b/litellm-rust/crates/auth-aws/src/aws.rs @@ -532,7 +532,7 @@ pub fn bedrock_model_id_and_region(model: &str) -> (String, Option) { (stripped.to_string(), region) } -fn is_bedrock_region(value: &str) -> bool { +pub fn is_bedrock_region(value: &str) -> bool { value.len() > 3 && value.contains('-') && value diff --git a/litellm-rust/crates/gateway-inference/src/responses.rs b/litellm-rust/crates/gateway-inference/src/responses.rs index f51066ed7f6..cf7d6db1dc4 100644 --- a/litellm-rust/crates/gateway-inference/src/responses.rs +++ b/litellm-rust/crates/gateway-inference/src/responses.rs @@ -36,6 +36,7 @@ pub(crate) async fn create( api_base: deployment.api_base.clone(), custom_llm_provider: deployment.custom_llm_provider.clone(), api_version: deployment.litellm_params.api_version.clone(), + litellm_params: deployment.litellm_params.clone(), extra_headers: None, timeout: deployment.timeout, }; diff --git a/litellm-rust/crates/inference-responses/Cargo.toml b/litellm-rust/crates/inference-responses/Cargo.toml index 793483ee76d..36076ddad8a 100644 --- a/litellm-rust/crates/inference-responses/Cargo.toml +++ b/litellm-rust/crates/inference-responses/Cargo.toml @@ -17,6 +17,7 @@ litellm-http.workspace = true litellm-inference.workspace = true litellm-llms.workspace = true litellm-llms-types.workspace = true +litellm-router-types.workspace = true litellm-secrets.workspace = true reqwest.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } diff --git a/litellm-rust/crates/inference-responses/src/prepare.rs b/litellm-rust/crates/inference-responses/src/prepare.rs index a846b64e7db..452af8d187a 100644 --- a/litellm-rust/crates/inference-responses/src/prepare.rs +++ b/litellm-rust/crates/inference-responses/src/prepare.rs @@ -5,6 +5,9 @@ use litellm_host::interceptors::{ProviderIdentity, RequestContext}; use litellm_llms::{ azure::responses::transformation::AZURE_RESPONSES_CONFIG, base_llm::responses::transformation::BaseResponsesApiConfig, + bedrock_mantle::responses::transformation::{ + BedrockMantleResponsesApiConfig, split_region_prefix, + }, databricks::responses::transformation::DATABRICKS_RESPONSES_CONFIG, fireworks_ai::responses::transformation::FIREWORKS_AI_RESPONSES_CONFIG, hosted_vllm::responses::transformation::HOSTED_VLLM_RESPONSES_CONFIG, @@ -12,6 +15,7 @@ use litellm_llms::{ openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG, xai::responses::transformation::XAI_RESPONSES_CONFIG, }; +use litellm_router_types::LitellmParams; use litellm_secrets::source::SecretSource; use serde_json::Value; @@ -25,8 +29,12 @@ pub(super) async fn prepare( call: ResponsesCall, secrets: &dyn SecretSource, ) -> Result { - let ResponsesProvider { identity, config } = - resolve_provider(&call.model, call.custom_llm_provider.as_deref())?; + let ResponsesProvider { identity, config } = resolve_provider( + &call.model, + call.custom_llm_provider.as_deref(), + call.api_base.as_deref(), + &call.litellm_params, + )?; let provider = identity.provider.as_str(); let model = identity.model.as_str(); let snapshot = secrets @@ -67,7 +75,7 @@ pub(super) async fn prepare( pub(super) struct ResponsesProvider { pub identity: ProviderIdentity, - pub config: &'static dyn BaseResponsesApiConfig, + pub config: Box, } /// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter @@ -75,6 +83,8 @@ pub(super) struct ResponsesProvider { pub(super) fn resolve_provider( model: &str, custom_llm_provider: Option<&str>, + api_base: Option<&str>, + params: &LitellmParams, ) -> Result { let explicit = custom_llm_provider.filter(|provider| !provider.is_empty()); let (provider, resolved) = match get_custom_llm_provider(model, explicit) { @@ -87,24 +97,34 @@ pub(super) fn resolve_provider( if resolved.is_empty() { return Err(Error::InvalidProvider(model.into())); } - let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) = - match provider.parse::() { - Ok(LlmProviders::Openai) if resolved.contains('/') => { - return Err(Error::InvalidProvider(model.into())); - } - Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()), - Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => ( - &OPENROUTER_RESPONSES_CONFIG, - format!("openrouter/{resolved}"), - ), - Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()), - Ok(LlmProviders::Azure) => (&AZURE_RESPONSES_CONFIG, resolved.into()), - Ok(LlmProviders::Xai) => (&XAI_RESPONSES_CONFIG, resolved.into()), - Ok(LlmProviders::Databricks) => (&DATABRICKS_RESPONSES_CONFIG, resolved.into()), - Ok(LlmProviders::FireworksAi) => (&FIREWORKS_AI_RESPONSES_CONFIG, resolved.into()), - Ok(LlmProviders::HostedVllm) => (&HOSTED_VLLM_RESPONSES_CONFIG, resolved.into()), - _ => return Err(Error::Unsupported("native HTTP responses provider")), - }; + let (config, wire_model): (Box, String) = match provider + .parse::( + ) { + Ok(LlmProviders::Openai) if resolved.contains('/') => { + return Err(Error::InvalidProvider(model.into())); + } + Ok(LlmProviders::Openai) => (Box::new(OpenAiResponsesApiConfig), resolved.into()), + Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => ( + Box::new(OPENROUTER_RESPONSES_CONFIG), + format!("openrouter/{resolved}"), + ), + Ok(LlmProviders::Openrouter) => (Box::new(OPENROUTER_RESPONSES_CONFIG), resolved.into()), + Ok(LlmProviders::Azure) => (Box::new(AZURE_RESPONSES_CONFIG), resolved.into()), + Ok(LlmProviders::Xai) => (Box::new(XAI_RESPONSES_CONFIG), resolved.into()), + Ok(LlmProviders::Databricks) => (Box::new(DATABRICKS_RESPONSES_CONFIG), resolved.into()), + Ok(LlmProviders::FireworksAi) => (Box::new(FIREWORKS_AI_RESPONSES_CONFIG), resolved.into()), + Ok(LlmProviders::HostedVllm) => (Box::new(HOSTED_VLLM_RESPONSES_CONFIG), resolved.into()), + Ok(LlmProviders::BedrockMantle) => { + let (region, wire) = split_region_prefix(resolved); + ( + Box::new(BedrockMantleResponsesApiConfig::new( + api_base, region, params, + )), + wire.into(), + ) + } + _ => return Err(Error::Unsupported("native HTTP responses provider")), + }; Ok(ResponsesProvider { identity: ProviderIdentity { model: wire_model, @@ -165,6 +185,12 @@ mod tests { "accounts/fireworks/models/qwen3" )] #[case::xai_prefix("xai/grok-4", None, "xai", "grok-4")] + #[case::bedrock_mantle_region_prefix( + "bedrock_mantle/us-west-2/openai.gpt-5.5", + None, + "bedrock_mantle", + "openai.gpt-5.5" + )] #[case::hosted_vllm_keeps_the_vendor( "hosted_vllm/meta-llama/Llama-3.1-8B", None, @@ -177,7 +203,8 @@ mod tests { #[case] provider: &str, #[case] wire_model: &str, ) { - let resolved = resolve_provider(model, custom_llm_provider).unwrap(); + let resolved = + resolve_provider(model, custom_llm_provider, None, &LitellmParams::default()).unwrap(); assert_eq!(resolved.identity.provider, provider); assert_eq!(resolved.identity.model, wire_model); } @@ -190,7 +217,7 @@ mod tests { #[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))] fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) { assert!(matches!( - resolve_provider(model, custom_llm_provider), + resolve_provider(model, custom_llm_provider, None, &LitellmParams::default()), Err(Error::InvalidProvider(_) | Error::Unsupported(_)) )); } diff --git a/litellm-rust/crates/inference-responses/src/types.rs b/litellm-rust/crates/inference-responses/src/types.rs index 473e28c1103..61dd985a56a 100644 --- a/litellm-rust/crates/inference-responses/src/types.rs +++ b/litellm-rust/crates/inference-responses/src/types.rs @@ -6,6 +6,7 @@ use litellm_llms::base_llm::{ auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig, }; use litellm_llms_types::formats::responses::ResponsesApiResponse; +use litellm_router_types::LitellmParams; use serde_json::{Map, Value}; use super::Error; @@ -18,6 +19,8 @@ pub struct ResponsesCall { pub api_base: Option, pub custom_llm_provider: Option, pub api_version: Option, + /// The deployment's provider settings, such as AWS credentials and region + pub litellm_params: LitellmParams, pub extra_headers: Option>, pub timeout: Option, } @@ -29,7 +32,7 @@ pub struct ResponsesStreamHead { pub type ResponsesOutput = CallOutput; pub(super) struct ProviderResponsesRequest { - pub config: &'static dyn BaseResponsesApiConfig, + pub config: Box, pub environment: ValidatedEnvironment, pub url: String, pub body: Value, diff --git a/litellm-rust/crates/inference-responses/tests/caching.rs b/litellm-rust/crates/inference-responses/tests/caching.rs index 0c887248da5..2583869ab62 100644 --- a/litellm-rust/crates/inference-responses/tests/caching.rs +++ b/litellm-rust/crates/inference-responses/tests/caching.rs @@ -285,6 +285,7 @@ async fn responses_cache_identity_follows_resolved_configuration_and_request_cal api_base: None, custom_llm_provider: None, api_version: None, + litellm_params: Default::default(), extra_headers: None, timeout: None, }, diff --git a/litellm-rust/crates/inference-responses/tests/live/openrouter.rs b/litellm-rust/crates/inference-responses/tests/live/openrouter.rs index b1fcf3084a5..4ac0104f220 100644 --- a/litellm-rust/crates/inference-responses/tests/live/openrouter.rs +++ b/litellm-rust/crates/inference-responses/tests/live/openrouter.rs @@ -34,6 +34,7 @@ fn call(stream: bool) -> ResponsesCall { api_base: None, custom_llm_provider: None, api_version: None, + litellm_params: Default::default(), extra_headers: None, timeout: Some(Duration::from_secs(60)), } diff --git a/litellm-rust/crates/inference-responses/tests/openrouter.rs b/litellm-rust/crates/inference-responses/tests/openrouter.rs index 0833cb0fce9..03984b71efa 100644 --- a/litellm-rust/crates/inference-responses/tests/openrouter.rs +++ b/litellm-rust/crates/inference-responses/tests/openrouter.rs @@ -23,6 +23,7 @@ fn call() -> ResponsesCall { api_base: None, custom_llm_provider: None, api_version: None, + litellm_params: Default::default(), extra_headers: None, timeout: None, } diff --git a/litellm-rust/crates/inference-responses/tests/responses.rs b/litellm-rust/crates/inference-responses/tests/responses.rs index acd2fd72e81..e5f35689558 100644 --- a/litellm-rust/crates/inference-responses/tests/responses.rs +++ b/litellm-rust/crates/inference-responses/tests/responses.rs @@ -28,6 +28,7 @@ fn call() -> ResponsesCall { api_base: None, custom_llm_provider: None, api_version: None, + litellm_params: Default::default(), extra_headers: None, timeout: None, } diff --git a/litellm-rust/crates/llms/AGENTS.md b/litellm-rust/crates/llms/AGENTS.md index 93b3d354fb0..8c27ed1afe3 100644 --- a/litellm-rust/crates/llms/AGENTS.md +++ b/litellm-rust/crates/llms/AGENTS.md @@ -58,7 +58,7 @@ - `AnthropicConfig`, `AmazonConverseConfig` and the non-OCR base traits are partial. Preserve their acceptance gates, passthrough and host fallback contracts - `OpenAiResponsesApiConfig` implements WebSocket transformations and direct HTTP Responses, without Python's model-specific param rewriting or Responses-to-Chat emulation -- OpenAI-compatible Responses hosts (`openrouter`, `azure`, `xai`, `hosted_vllm`, `fireworks_ai`, `databricks`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules +- OpenAI-compatible Responses hosts (`openrouter`, `azure`, `xai`, `hosted_vllm`, `fireworks_ai`, `databricks`, `bedrock_mantle`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules - Chat `supported_openai_param_mappings` lists OpenAI/provider name pairs, unlike Python's name-only list. Audio `map_transcription_params` is a Rust filtering helper - Auth resolution is split between configs and route preparation in `inference-` diff --git a/litellm-rust/crates/llms/src/base_llm/responses/additional_tools.rs b/litellm-rust/crates/llms/src/base_llm/responses/additional_tools.rs new file mode 100644 index 00000000000..354e086b762 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/responses/additional_tools.rs @@ -0,0 +1,72 @@ +//! Tools a client sends as `additional_tools` input items, matching +//! `litellm/responses/additional_tools.py` + +use serde_json::Value; + +pub struct HoistedAdditionalTools { + pub input: Value, + pub hoisted: Vec, +} + +/// The input without its `additional_tools` items, and the tools those items carried. An item +/// whose `tools` is not a list of objects carries none +pub fn hoist_additional_tools(input: Value) -> HoistedAdditionalTools { + let Value::Array(items) = input else { + return HoistedAdditionalTools { + input, + hoisted: Vec::new(), + }; + }; + let (additional, remaining): (Vec, Vec) = items + .into_iter() + .partition(|item| item.get("type").and_then(Value::as_str) == Some("additional_tools")); + HoistedAdditionalTools { + input: Value::Array(remaining), + hoisted: additional.into_iter().flat_map(item_tools).collect(), + } +} + +fn item_tools(item: Value) -> Vec { + match item.get("tools") { + None => Vec::new(), + Some(Value::Array(tools)) if tools.iter().all(Value::is_object) => tools.clone(), + Some(_) => Vec::new(), + } +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + #[rstest] + #[case::tools_are_hoisted_in_order( + json!([ + {"type": "additional_tools", "tools": [{"type": "function", "name": "a"}]}, + {"role": "user", "content": "hi"}, + {"type": "additional_tools", "tools": [{"type": "mcp", "server_label": "b"}]}, + ]), + json!([{"role": "user", "content": "hi"}]), + vec![json!({"type": "function", "name": "a"}), json!({"type": "mcp", "server_label": "b"})] + )] + #[case::malformed_tools_carry_nothing_but_the_item_still_leaves( + json!([ + {"type": "additional_tools", "tools": [{"type": "function"}, "bad"]}, + {"type": "additional_tools", "tools": null}, + {"type": "additional_tools"}, + ]), + json!([]), + vec![] + )] + #[case::string_input(json!("hi"), json!("hi"), vec![])] + fn additional_tools_items_are_hoisted( + #[case] input: Value, + #[case] remaining: Value, + #[case] hoisted: Vec, + ) { + let result = hoist_additional_tools(input); + assert_eq!((result.input, result.hoisted), (remaining, hoisted)); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/responses/codex_compat.rs b/litellm-rust/crates/llms/src/base_llm/responses/codex_compat.rs new file mode 100644 index 00000000000..18dd166dc27 --- /dev/null +++ b/litellm-rust/crates/llms/src/base_llm/responses/codex_compat.rs @@ -0,0 +1,152 @@ +//! Codex CLI wire-format quirks shared by the Responses providers that need them, matching +//! `litellm/llms/base_llm/responses/codex_compat.py` + +use litellm_python_compat::{ + json::{dumps, from_json}, + repr::to_str, + truthy::truthy, +}; +use serde_json::{Map, Value, json}; + +/// Codex history item types that api.openai.com accepts and other Responses backends reject, +/// rewritten to the spelling those backends take. An item that carries nothing to rewrite is +/// dropped +pub fn normalize_codex_input_items(input: Value) -> Value { + match input { + Value::Array(items) => Value::Array(items.into_iter().filter_map(normalized).collect()), + other => other, + } +} + +/// The tools whose `type` the backend accepts; a tool that is not an object passes through +pub fn drop_unsupported_tools(tools: Vec, supported: &[&str]) -> Vec { + tools + .into_iter() + .filter(|tool| match tool { + Value::Object(fields) => fields + .get("type") + .and_then(Value::as_str) + .is_some_and(|kind| supported.contains(&kind)), + _ => true, + }) + .collect() +} + +fn normalized(item: Value) -> Option { + let Value::Object(fields) = &item else { + return Some(item); + }; + match fields.get("type").and_then(Value::as_str) { + Some("agent_message") => agent_message(fields), + Some("context_compaction") => context_compaction(fields), + Some("local_shell_call") => local_shell_call(fields), + _ => Some(item), + } +} + +fn agent_message(fields: &Map) -> Option { + let text: String = fields + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(Value::as_object) + .map(|block| { + ["text", "encrypted_content"] + .iter() + .filter_map(|name| block.get(*name)) + .map(|value| from_json(value.clone())) + .find(truthy) + .map(|value| to_str(&value)) + .unwrap_or_default() + }) + .collect(); + (!text.is_empty()).then(|| { + json!({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]}) + }) +} + +fn context_compaction(fields: &Map) -> Option { + fields + .get("encrypted_content") + .and_then(Value::as_str) + .filter(|content| !content.is_empty()) + .map(|content| json!({"type": "compaction", "encrypted_content": content})) +} + +fn local_shell_call(fields: &Map) -> Option { + let call_id = fields + .get("call_id") + .and_then(Value::as_str) + .filter(|call_id| !call_id.is_empty())?; + let arguments = match fields.get("action") { + Some(action @ Value::Object(_)) => dumps(&from_json(action.clone())).ok()?, + _ => "{}".to_owned(), + }; + Some(json!({ + "type": "function_call", + "call_id": call_id, + "name": "local_shell", + "arguments": arguments, + })) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + + use super::*; + + #[rstest] + #[case::agent_message_joins_text_and_plaintext_args( + json!({"type": "agent_message", "content": [{"text": "a"}, {"encrypted_content": "b"}, {"text": "", "encrypted_content": "c"}, "skipped"]}), + Some(json!({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "abc"}]})) + )] + #[case::agent_message_renders_non_string_text_like_python( + json!({"type": "agent_message", "content": [{"text": 1.0}, {"text": true}]}), + Some(json!({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "1.0True"}]})) + )] + #[case::empty_agent_message_is_dropped(json!({"type": "agent_message", "content": [{"text": ""}]}), None)] + #[case::context_compaction( + json!({"type": "context_compaction", "encrypted_content": "enc"}), + Some(json!({"type": "compaction", "encrypted_content": "enc"})) + )] + #[case::empty_context_compaction_is_dropped(json!({"type": "context_compaction", "encrypted_content": ""}), None)] + #[case::local_shell_call_dumps_its_action_like_python( + json!({"type": "local_shell_call", "call_id": "c1", "action": {"command": ["ls", "é"], "timeout": 1}}), + Some(json!({"type": "function_call", "call_id": "c1", "name": "local_shell", "arguments": "{\"command\": [\"ls\", \"\\u00e9\"], \"timeout\": 1}"})) + )] + #[case::local_shell_call_without_action( + json!({"type": "local_shell_call", "call_id": "c1", "action": "ls"}), + Some(json!({"type": "function_call", "call_id": "c1", "name": "local_shell", "arguments": "{}"})) + )] + #[case::local_shell_call_without_call_id_is_dropped(json!({"type": "local_shell_call", "call_id": ""}), None)] + #[case::other_items_pass_through(json!({"type": "message", "role": "user", "content": "hi"}), Some(json!({"type": "message", "role": "user", "content": "hi"})))] + fn codex_items_take_the_backend_spelling(#[case] item: Value, #[case] expected: Option) { + assert_eq!( + normalize_codex_input_items(json!([item])), + Value::Array(expected.into_iter().collect()) + ); + } + + #[test] + fn string_input_is_untouched() { + assert_eq!(normalize_codex_input_items(json!("hi")), json!("hi")); + } + + #[test] + fn unsupported_tool_types_are_dropped() { + assert_eq!( + drop_unsupported_tools( + vec![ + json!({"type": "function", "name": "f"}), + json!({"type": "web_search"}), + json!({"name": "untyped"}), + json!("opaque"), + ], + &["function"], + ), + vec![json!({"type": "function", "name": "f"}), json!("opaque")] + ); + } +} diff --git a/litellm-rust/crates/llms/src/base_llm/responses/mod.rs b/litellm-rust/crates/llms/src/base_llm/responses/mod.rs index fa7df180f50..d5ca0ebc85d 100644 --- a/litellm-rust/crates/llms/src/base_llm/responses/mod.rs +++ b/litellm-rust/crates/llms/src/base_llm/responses/mod.rs @@ -1,2 +1,4 @@ +pub mod additional_tools; +pub mod codex_compat; pub mod streaming; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/mod.rs b/litellm-rust/crates/llms/src/bedrock_mantle/mod.rs index ba63992f3cb..e4838133a04 100644 --- a/litellm-rust/crates/llms/src/bedrock_mantle/mod.rs +++ b/litellm-rust/crates/llms/src/bedrock_mantle/mod.rs @@ -1 +1,2 @@ pub mod messages; +pub mod responses; diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/responses/AGENTS.md b/litellm-rust/crates/llms/src/bedrock_mantle/responses/AGENTS.md new file mode 100644 index 00000000000..658250b8515 --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock_mantle/responses/AGENTS.md @@ -0,0 +1,12 @@ +# rules + +- Send the OpenAI Responses payload to `/openai/v1/responses` when the host passes `use_openai_responses_path` from the model cost map, else `/v1/responses` +- A standard `bedrock-mantle..api.aws` base is re-pinned to the resolved region, and any other base keeps its path +- Authenticate with a bearer from `api_key`, `BEDROCK_MANTLE_API_KEY` or `AWS_BEARER_TOKEN_BEDROCK`, otherwise sign with AWS SigV4 for service `bedrock` +- Keep only the tool types Mantle accepts, hoist `additional_tools` input items into `tools`, and rewrite Codex history items +- Models that serve only Chat Completions, and `service_tier` or `reasoning.summary` values Mantle rejects, stay on Python +- No native WebSocket mode + +# references + +- Python counterpart `litellm/llms/bedrock_mantle/responses/transformation.py` diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/responses/mod.rs b/litellm-rust/crates/llms/src/bedrock_mantle/responses/mod.rs new file mode 100644 index 00000000000..f239b6921fa --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock_mantle/responses/mod.rs @@ -0,0 +1 @@ +pub mod transformation; diff --git a/litellm-rust/crates/llms/src/bedrock_mantle/responses/transformation.rs b/litellm-rust/crates/llms/src/bedrock_mantle/responses/transformation.rs new file mode 100644 index 00000000000..742bf96ab22 --- /dev/null +++ b/litellm-rust/crates/llms/src/bedrock_mantle/responses/transformation.rs @@ -0,0 +1,504 @@ +//! Bedrock Mantle Responses, matching `litellm/llms/bedrock_mantle/responses/transformation.py` + +use std::sync::LazyLock; + +use litellm_auth::{AwsParams, CredentialPlacement, SecretValue}; +use litellm_auth_aws::{AwsCredentialSource, is_bedrock_region}; +use litellm_http::request::{with_header, without_headers}; +use litellm_llms_types::formats::responses::ResponsesApiResponse; +use litellm_python_compat::truthy::json_truthy; +use litellm_router_types::LitellmParams; +use serde_json::{Map, Value}; + +use crate::{ + Error, + base_llm::{ + auth::{AuthScheme, ValidatedEnvironment}, + responses::{ + additional_tools::hoist_additional_tools, + codex_compat::{drop_unsupported_tools, normalize_codex_input_items}, + transformation::BaseResponsesApiConfig, + }, + }, + bedrock::common_utils::validated_region, + openai::responses::{ + request::ResponsesHost, + transformation::{responses_request, responses_response}, + }, +}; + +const KEY_ENVS: [&str; 2] = ["BEDROCK_MANTLE_API_KEY", "AWS_BEARER_TOKEN_BEDROCK"]; +const API_BASE_ENV: &str = "BEDROCK_MANTLE_API_BASE"; +const REGION_ENVS: [&str; 3] = ["BEDROCK_MANTLE_REGION", "AWS_REGION_NAME", "AWS_REGION"]; +const DEFAULT_REGION: &str = "us-east-1"; +const BASE_SUFFIXES: [&str; 5] = [ + "/openai/v1/responses", + "/v1/responses", + "/responses", + "/openai/v1", + "/v1", +]; +/// Per Bedrock Mantle Responses API validation errors +const SUPPORTED_TOOL_TYPES: [&str; 6] = [ + "function", + "mcp", + "custom", + "namespace", + "tool_search", + "web_search", +]; +/// The price map's `use_openai_responses_path`, which the host reads for the model +const OPENAI_PATH_PARAM: &str = "use_openai_responses_path"; +const PROJECT_PARAM: &str = "aws_bedrock_project_id"; + +/// One call's Mantle connection, since the model picks the Responses path +pub struct BedrockMantleResponsesApiConfig { + api_base: Option, + model_region: Option, + aws: AwsParams, + project_id: Option, + openai_path: bool, +} + +impl BedrockMantleResponsesApiConfig { + pub fn new(api_base: Option<&str>, model_region: Option<&str>, params: &LitellmParams) -> Self { + Self { + api_base: api_base.map(str::to_owned), + model_region: model_region.map(str::to_owned), + aws: params.aws.clone(), + project_id: params + .extra + .get(PROJECT_PARAM) + .and_then(Value::as_str) + .filter(|project| !project.is_empty()) + .map(str::to_owned), + openai_path: params.extra.get(OPENAI_PATH_PARAM) == Some(&Value::Bool(true)), + } + } + + /// Python's `resolve_mantle_region`; the model's region prefix stands in for the default + /// base `get_llm_provider` builds from it + fn region( + &self, + api_base: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + if let Some(region) = non_empty(self.aws.aws_region_name.as_deref()) { + return validated_region(region.to_owned()); + } + let routed = match configured_base(api_base, lookup) { + Some(base) => mantle_host_region(base.trim_end_matches('/')).map(str::to_owned), + None => self.model_region.clone(), + }; + validated_region( + routed + .or_else(|| { + REGION_ENVS + .iter() + .find_map(|name| lookup(name).filter(|value| !value.is_empty())) + }) + .unwrap_or_else(|| DEFAULT_REGION.into()), + ) + } +} + +/// A leading Bedrock region in a Mantle model id routes the call and is not sent +pub fn split_region_prefix(model: &str) -> (Option<&str>, &str) { + match model.split_once('/') { + Some((region, rest)) if is_bedrock_region(region) => (Some(region), rest), + _ => (None, model), + } +} + +impl BaseResponsesApiConfig for BedrockMantleResponsesApiConfig { + fn secret_names( + &self, + _api_key: Option<&str>, + _api_base: Option<&str>, + ) -> &'static [&'static str] { + static NAMES: LazyLock> = LazyLock::new(|| { + KEY_ENVS + .into_iter() + .chain([API_BASE_ENV]) + .chain(REGION_ENVS) + .chain( + AwsParams::secret_names() + .iter() + .copied() + .filter(|name| !REGION_ENVS.contains(name)), + ) + .collect() + }); + &NAMES + } + + fn validate_environment( + &self, + headers: Vec<(String, String)>, + api_key: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let headers = with_header(headers, "content-type", "application/json".into()); + let headers = match &self.project_id { + Some(project) => with_header(headers, "OpenAI-Project", project.clone()), + None => headers, + }; + let bearer = non_empty(api_key).map(str::to_owned).or_else(|| { + KEY_ENVS + .iter() + .find_map(|name| lookup(name).filter(|value| !value.is_empty())) + }); + Ok(match bearer { + Some(token) => ValidatedEnvironment { + headers, + auth: AuthScheme::Credential { + placement: CredentialPlacement::Bearer, + secret: SecretValue::new(token), + }, + }, + None => ValidatedEnvironment { + headers: without_headers(headers, &["authorization"]), + auth: AuthScheme::AwsSigV4 { + region: self.region(self.api_base.as_deref(), lookup)?, + service: "bedrock", + credentials: Box::new(AwsCredentialSource::from_params(&self.aws, lookup)), + }, + }, + }) + } + + fn get_complete_url( + &self, + api_base: Option<&str>, + _api_version: Option<&str>, + lookup: &dyn Fn(&str) -> Option, + ) -> Result { + let region = self.region(api_base, lookup)?; + let default = format!("https://bedrock-mantle.{region}.api.aws"); + let configured = configured_base(api_base, lookup).unwrap_or_else(|| default.clone()); + let trimmed = configured.trim_end_matches('/'); + let stripped = BASE_SUFFIXES + .iter() + .find_map(|suffix| trimmed.strip_suffix(suffix)) + .unwrap_or(trimmed); + let base = match mantle_host_region(stripped) { + Some(_) => default.as_str(), + None => stripped, + }; + let path = match self.openai_path { + true => "/openai/v1/responses", + false => "/v1/responses", + }; + Ok(format!("{base}{path}")) + } + + fn transform_responses_api_request( + &self, + model: &str, + input: Value, + params: Map, + ) -> Result { + let params = with_supported_tools(params); + let hoisted = hoist_additional_tools(input); + let params = match hoisted.hoisted.is_empty() { + true => params, + false => with_hoisted_tools(params, hoisted.hoisted), + }; + responses_request( + model, + normalize_codex_input_items(hoisted.input), + params, + ResponsesHost::OpenAi, + ) + } + + fn transform_response_api_response(&self, body: Value) -> Result { + responses_response(body) + } +} + +fn non_empty(value: Option<&str>) -> Option<&str> { + value.filter(|value| !value.is_empty()) +} + +fn configured_base( + api_base: Option<&str>, + lookup: &dyn Fn(&str) -> Option, +) -> Option { + non_empty(api_base) + .map(str::to_owned) + .or_else(|| lookup(API_BASE_ENV).filter(|base| !base.is_empty())) +} + +/// Python's `MANTLE_HOST_RE`: `^https?://bedrock-mantle\.([^/.]+)\.api\.aws(?=/|$)`, ignoring case +fn mantle_host_region(base: &str) -> Option<&str> { + let lowered = base.to_ascii_lowercase(); + let scheme = ["https://", "http://"] + .into_iter() + .find(|scheme| lowered.starts_with(scheme))?; + let host = &base[scheme.len()..]; + let region_start = "bedrock-mantle.".len(); + if !host + .get(..region_start)? + .eq_ignore_ascii_case("bedrock-mantle.") + { + return None; + } + let after = &host[region_start..]; + let region_end = after.find(['.', '/']).filter(|end| *end > 0)?; + let rest = &after[region_end..]; + let suffix = ".api.aws"; + let matched = rest.get(..suffix.len())?.eq_ignore_ascii_case(suffix) + && matches!(rest.as_bytes().get(suffix.len()), None | Some(b'/')); + matched.then(|| &after[..region_end]) +} + +/// Python's `map_openai_params` tool filter: a single tool is listed, and no supported tool +/// leaves no `tools` +fn with_supported_tools(mut params: Map) -> Map { + if !params.get("tools").is_some_and(json_truthy) { + return params; + } + let listed = match params.remove("tools") { + Some(Value::Array(tools)) => tools, + tool => tool.into_iter().collect(), + }; + with_tools( + params, + drop_unsupported_tools(listed, &SUPPORTED_TOOL_TYPES), + ) +} + +fn with_hoisted_tools(mut params: Map, hoisted: Vec) -> Map { + let existing = match params.remove("tools") { + Some(Value::Array(tools)) => tools, + _ => Vec::new(), + }; + let combined = existing.into_iter().chain(hoisted).collect(); + with_tools( + params, + drop_unsupported_tools(combined, &SUPPORTED_TOOL_TYPES), + ) +} + +fn with_tools(mut params: Map, tools: Vec) -> Map { + if !tools.is_empty() { + params.insert("tools".into(), Value::Array(tools)); + } + params +} + +#[cfg(test)] +mod tests { + use litellm_router_types::LitellmParams; + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option { + move |name| { + pairs + .iter() + .find(|(key, _)| *key == name) + .map(|(_, value)| (*value).to_owned()) + } + } + + fn params(mut fields: Value) -> LitellmParams { + fields["model"] = json!("m"); + serde_json::from_value(fields).unwrap() + } + + fn config( + api_base: Option<&str>, + model_region: Option<&str>, + fields: Value, + ) -> BedrockMantleResponsesApiConfig { + BedrockMantleResponsesApiConfig::new(api_base, model_region, ¶ms(fields)) + } + + #[rstest] + #[case::default_region(None, None, json!({}), &[], "https://bedrock-mantle.us-east-1.api.aws/v1/responses")] + #[case::openai_path(None, None, json!({"use_openai_responses_path": true}), &[], "https://bedrock-mantle.us-east-1.api.aws/openai/v1/responses")] + #[case::model_prefix_region(None, Some("us-west-2"), json!({}), &[], "https://bedrock-mantle.us-west-2.api.aws/v1/responses")] + #[case::region_env(None, None, json!({}), &[("AWS_REGION", "eu-west-1")], "https://bedrock-mantle.eu-west-1.api.aws/v1/responses")] + #[case::aws_region_name_repins_a_mantle_host( + Some("https://bedrock-mantle.us-east-1.api.aws/openai/v1"), + None, + json!({"aws_region_name": "us-west-2", "use_openai_responses_path": true}), + &[], + "https://bedrock-mantle.us-west-2.api.aws/openai/v1/responses" + )] + #[case::mantle_host_region_beats_the_env( + Some("https://Bedrock-Mantle.eu-central-1.api.aws/v1/responses/"), + None, + json!({}), + &[("BEDROCK_MANTLE_REGION", "us-west-2")], + "https://bedrock-mantle.eu-central-1.api.aws/v1/responses" + )] + #[case::custom_proxy_keeps_its_path( + Some("https://proxy.test/tenant/v1"), + Some("us-west-2"), + json!({}), + &[], + "https://proxy.test/tenant/v1/responses" + )] + #[case::base_secret(None, None, json!({}), &[("BEDROCK_MANTLE_API_BASE", "https://proxy.test")], "https://proxy.test/v1/responses")] + fn url_matches_python( + #[case] api_base: Option<&str>, + #[case] model_region: Option<&str>, + #[case] fields: Value, + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + assert_eq!( + config(api_base, model_region, fields) + .get_complete_url(api_base, None, &env(secrets)) + .unwrap(), + expected + ); + } + + #[rstest] + #[case::https("https://bedrock-mantle.us-east-1.api.aws", Some("us-east-1"))] + #[case::path("http://bedrock-mantle.us-east-1.api.aws/tenant", Some("us-east-1"))] + #[case::port("https://bedrock-mantle.us-east-1.api.aws:443", None)] + #[case::other_domain("https://bedrock-mantle.us-east-1.api.aws.test", None)] + #[case::empty_region("https://bedrock-mantle..api.aws", None)] + #[case::proxy("https://proxy.test", None)] + fn mantle_hosts_match_pythons_pattern(#[case] base: &str, #[case] expected: Option<&str>) { + assert_eq!(mantle_host_region(base), expected); + } + + #[rstest] + #[case::regional("us-west-2/openai.gpt-5.5", Some("us-west-2"), "openai.gpt-5.5")] + #[case::gov( + "us-gov-west-1/openai.gpt-5.4", + Some("us-gov-west-1"), + "openai.gpt-5.4" + )] + #[case::bare("openai.gpt-5.5", None, "openai.gpt-5.5")] + fn region_prefixes_route_and_are_not_sent( + #[case] model: &str, + #[case] region: Option<&str>, + #[case] wire: &str, + ) { + assert_eq!(split_region_prefix(model), (region, wire)); + } + + #[rstest] + #[case::explicit(Some("explicit"), &[("BEDROCK_MANTLE_API_KEY", "env")], "explicit")] + #[case::mantle_secret(None, &[("BEDROCK_MANTLE_API_KEY", "mantle"), ("AWS_BEARER_TOKEN_BEDROCK", "aws")], "mantle")] + #[case::bedrock_secret(Some(""), &[("AWS_BEARER_TOKEN_BEDROCK", "aws")], "aws")] + fn bearer_tokens_skip_sigv4( + #[case] api_key: Option<&str>, + #[case] secrets: &'static [(&'static str, &'static str)], + #[case] expected: &str, + ) { + let environment = config(None, None, json!({})) + .validate_environment(vec![], api_key, &env(secrets)) + .unwrap(); + let AuthScheme::Credential { placement, secret } = environment.auth else { + panic!("expected a bearer credential"); + }; + assert_eq!( + (placement, secret.expose()), + (CredentialPlacement::Bearer, expected) + ); + } + + #[test] + fn without_a_bearer_the_call_is_sigv4_signed_for_the_url_region() { + let environment = config( + Some("https://bedrock-mantle.eu-west-1.api.aws/openai/v1"), + Some("us-west-2"), + json!({"aws_bedrock_project_id": "proj_1"}), + ) + .validate_environment( + vec![("Authorization".into(), "Bearer caller".into())], + None, + &env(&[]), + ) + .unwrap(); + assert!(matches!( + environment.auth, + AuthScheme::AwsSigV4 { ref region, service: "bedrock", .. } if region == "eu-west-1" + )); + assert_eq!( + environment.headers, + vec![ + ("content-type".to_owned(), "application/json".to_owned()), + ("openai-project".to_owned(), "proj_1".to_owned()), + ] + ); + } + + #[test] + fn an_invalid_region_is_a_request_error() { + assert!(matches!( + config(None, None, json!({"aws_region_name": "US EAST"})).get_complete_url( + None, + None, + &env(&[]) + ), + Err(Error::InvalidRequest(_)) + )); + } + + #[rstest] + #[case::unsupported_dropped( + json!([{"type": "function", "name": "f"}, {"type": "file_search"}]), + json!("hi"), + Some(json!([{"type": "function", "name": "f"}])) + )] + #[case::single_tool_listed(json!({"type": "web_search"}), json!("hi"), Some(json!([{"type": "web_search"}])))] + #[case::none_supported(json!([{"type": "image_generation"}]), json!("hi"), None)] + #[case::empty_list_kept(json!([]), json!("hi"), Some(json!([])))] + #[case::hoisted_after_existing( + json!([{"type": "function", "name": "f"}]), + json!([{"type": "additional_tools", "tools": [{"type": "mcp", "server_label": "s"}, {"type": "code_interpreter"}]}, {"role": "user", "content": "hi"}]), + Some(json!([{"type": "function", "name": "f"}, {"type": "mcp", "server_label": "s"}])) + )] + #[case::hoisted_into_empty_tools( + json!([]), + json!([{"type": "additional_tools", "tools": [{"type": "code_interpreter"}]}, {"role": "user", "content": "hi"}]), + None + )] + fn tools_are_filtered_to_what_mantle_accepts( + #[case] tools: Value, + #[case] input: Value, + #[case] expected: Option, + ) { + let body = config(None, None, json!({})) + .transform_responses_api_request( + "openai.gpt-5.5", + input, + Map::from_iter([("tools".into(), tools)]), + ) + .unwrap(); + assert_eq!(body.get("tools").cloned(), expected); + } + + #[test] + fn codex_history_is_rewritten_after_hoisting() { + let body = config(None, None, json!({})) + .transform_responses_api_request( + "openai.gpt-5.5", + json!([ + {"type": "additional_tools", "tools": [{"type": "function", "name": "f"}]}, + {"type": "context_compaction", "encrypted_content": "enc"}, + {"role": "user", "content": "hi"}, + ]), + Map::new(), + ) + .unwrap(); + assert_eq!( + (&body["input"], &body["tools"]), + ( + &json!([{"type": "compaction", "encrypted_content": "enc"}, {"role": "user", "content": "hi"}]), + &json!([{"type": "function", "name": "f"}]) + ) + ); + } +} diff --git a/litellm-rust/crates/llms/src/xai/responses/transformation.rs b/litellm-rust/crates/llms/src/xai/responses/transformation.rs index 67dfbab913e..7cb8b9ceac6 100644 --- a/litellm-rust/crates/llms/src/xai/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/xai/responses/transformation.rs @@ -2,6 +2,7 @@ use litellm_auth::{CredentialPlacement, SecretValue}; use litellm_llms_types::formats::responses::ResponsesApiResponse; +use litellm_python_compat::truthy::json_truthy; use serde_json::{Map, Value, json}; use crate::{ @@ -89,7 +90,7 @@ impl BaseResponsesApiConfig for XaiResponsesApiConfig { mut params: Map, ) -> Result { params.remove("metadata"); - if let Some(tools) = params.get_mut("tools").filter(|tools| python_truthy(tools)) { + if let Some(tools) = params.get_mut("tools").filter(|tools| json_truthy(tools)) { let listed = match tools.take() { Value::Array(tools) => tools, tool => vec![tool], @@ -122,17 +123,6 @@ fn resolved( .or_else(|| lookup(name).filter(|value| !value.is_empty())) } -fn python_truthy(value: &Value) -> bool { - match value { - Value::Null => false, - Value::Bool(value) => *value, - Value::Number(number) => number.as_f64() != Some(0.0), - Value::String(value) => !value.is_empty(), - Value::Array(values) => !values.is_empty(), - Value::Object(values) => !values.is_empty(), - } -} - /// Python's `xai_reported_cost_in_usd`: only a non-negative integer tick count is a cost fn reported_cost(ticks: Option<&Value>) -> Option { ticks? diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs index 035bab7575d..f660530713c 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/host.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/host.rs @@ -1,6 +1,7 @@ use std::convert::Infallible; -use crate::routes::inference::InferenceHost; +use crate::{marshal::project_optional_fields, routes::inference::InferenceHost}; +use litellm_auth::AwsParams; use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider; use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py}; use litellm_inference_responses::{ @@ -9,7 +10,9 @@ use litellm_inference_responses::{ route::Responses, types::ResponsesCall, }; +use litellm_router_types::LitellmParams; use pyo3::{ + exceptions::PyValueError, gc::{PyTraverseError, PyVisit}, prelude::*, types::PyDict, @@ -53,6 +56,15 @@ pub(super) fn project( .or(raw("base_url")?), ))? .extract()?; + let litellm_params = litellm_params( + py, + &call.options.model, + call.options.custom_llm_provider.as_deref(), + project_optional_fields( + AwsParams::fields().chain(["aws_bedrock_project_id"]), + |name| host.argument(py, arguments, name), + )?, + )?; let optional_params = call .params .into_iter() @@ -89,11 +101,36 @@ pub(super) fn project( .argument(py, arguments, "api_version")? .map(|value| value.extract()) .transpose()?, + litellm_params, extra_headers: call.options.extra_headers, timeout: call.options.timeout, }) } +/// The caller's AWS settings plus what Python reads from the model cost map for the provider +fn litellm_params( + py: Python<'_>, + model: &str, + custom_llm_provider: Option<&str>, + fields: serde_json::Map, +) -> PyResult { + let provider_params: Vec<(String, serde_json::Value)> = from_py( + &py.import(super::ROUTE_HOST_MODULE)? + .getattr("provider_params")? + .call1((model, custom_llm_provider))?, + )?; + let fields = [( + "model".to_owned(), + serde_json::Value::String(model.to_owned()), + )] + .into_iter() + .chain(fields) + .chain(provider_params) + .collect(); + serde_json::from_value(serde_json::Value::Object(fields)) + .map_err(|error| PyValueError::new_err(format!("invalid litellm params: {error}"))) +} + /// The provider and deployment Python encodes into the ids it returns: the resolved provider and /// `litellm_metadata.model_info.id` fn id_scope( diff --git a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs index 41973795ca1..fe7e4ef88bb 100644 --- a/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs +++ b/litellm-rust/crates/python-bridge/src/routes/responses/mod.rs @@ -47,7 +47,9 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py None => !model.contains('/'), Some(resolved) => match resolved.custom_llm_provider { "openai" => !resolved.model.contains('/'), - "azure" | "xai" | "hosted_vllm" | "fireworks_ai" | "databricks" => true, + "azure" | "xai" | "hosted_vllm" | "fireworks_ai" | "databricks" | "bedrock_mantle" => { + true + } _ => false, }, }; diff --git a/litellm-rust/crates/python-compat/src/truthy.rs b/litellm-rust/crates/python-compat/src/truthy.rs index af747202572..c5cb57d2bea 100644 --- a/litellm-rust/crates/python-compat/src/truthy.rs +++ b/litellm-rust/crates/python-compat/src/truthy.rs @@ -16,3 +16,15 @@ pub fn truthy(value: &Value) -> bool { Value::Dict(entries) => !entries.is_empty(), } } + +/// [`truthy`] for the value `json.loads` returns, without converting it +pub fn json_truthy(value: &serde_json::Value) -> bool { + match value { + serde_json::Value::Null => false, + serde_json::Value::Bool(value) => *value, + serde_json::Value::Number(number) => number.as_f64() != Some(0.0), + serde_json::Value::String(value) => !value.is_empty(), + serde_json::Value::Array(values) => !values.is_empty(), + serde_json::Value::Object(values) => !values.is_empty(), + } +} diff --git a/litellm/rust_bridge/host/routes/responses.py b/litellm/rust_bridge/host/routes/responses.py index f975cca69d5..1d68a4ec230 100644 --- a/litellm/rust_bridge/host/routes/responses.py +++ b/litellm/rust_bridge/host/routes/responses.py @@ -8,6 +8,7 @@ from typing import Final, cast # noqa: TID251 # narrows caller json the checke import litellm from litellm import get_llm_provider from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig +from litellm.llms.bedrock_mantle.responses.transformation import BedrockMantleResponsesAPIConfig from litellm.llms.databricks.responses.transformation import DatabricksResponsesAPIConfig from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id from litellm.llms.fireworks_ai.responses.transformation import FireworksAIResponsesAPIConfig @@ -19,9 +20,25 @@ from litellm.rust_bridge.public_call import inference_decline_reason, known_para from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse from litellm.types.router import GenericLiteLLMParams +from litellm.types.utils import LlmProviders +from litellm.utils import ProviderConfigManager PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__) _NATIVE_PARAMETERS: Final = (*PARAMETERS, "api_version") +_MANTLE_PARAMETERS: Final = ( + *_NATIVE_PARAMETERS, + "aws_access_key_id", + "aws_secret_access_key", + "aws_session_token", + "aws_region_name", + "aws_session_name", + "aws_profile_name", + "aws_role_name", + "aws_web_identity_token", + "aws_sts_endpoint", + "aws_external_id", + "aws_bedrock_project_id", +) _CONFIGS: Final[Mapping[str, OpenAIResponsesAPIConfig]] = MappingProxyType( { "openai": OpenAIResponsesAPIConfig(), @@ -40,7 +57,7 @@ def body_params(request: Mapping[str, object]) -> Mapping[str, object]: def connection_defaults(provider: str) -> tuple[str | None, str | None]: - if provider == "databricks": + if provider in ("databricks", "bedrock_mantle"): return None, None provider_key: Final = {"openai": litellm.openai_key, "azure": litellm.azure_key}.get(provider) return litellm.api_key or provider_key, litellm.api_base @@ -56,6 +73,21 @@ def provider_connection( return provider_key, provider_base +def provider_params(model: str, custom_llm_provider: str | None) -> tuple[tuple[str, object], ...]: + """The provider settings Python reads from the model cost map, as litellm params""" + resolved, provider = _resolved_model({"model": model, "custom_llm_provider": custom_llm_provider}) + if provider != "bedrock_mantle": + return () + config: Final = _mantle_config(resolved) + return () if config is None else (("use_openai_responses_path", config.use_openai_path),) + + +def _mantle_config(model: str) -> BedrockMantleResponsesAPIConfig | None: + """The config Python's dispatch picks from the model cost map, or None for chat-only models""" + config: Final = ProviderConfigManager.get_provider_responses_api_config(LlmProviders.BEDROCK_MANTLE, model) + return config if isinstance(config, BedrockMantleResponsesAPIConfig) else None + + def response(value: Mapping[str, object]) -> ResponsesAPIResponse: return ResponsesAPIResponse.model_validate(value) @@ -84,16 +116,22 @@ def decline_reason(request: Mapping[str, object]) -> str | None: model, provider = _resolved_model(request) except litellm.exceptions.BadRequestError: return "native Responses could not resolve the provider" - config: Final = _CONFIGS.get(provider) + config: Final = _responses_config(provider, model) if config is None: return "native HTTP responses provider" return ( - inference_decline_reason(_NATIVE_PARAMETERS, request) + inference_decline_reason(_MANTLE_PARAMETERS if provider == "bedrock_mantle" else _NATIVE_PARAMETERS, request) or _provider_decline_reason(provider, model, request) or _model_rules_decline_reason(config, model, request) ) +def _responses_config(provider: str, model: str) -> OpenAIResponsesAPIConfig | None: + if provider != "bedrock_mantle": + return _CONFIGS.get(provider) + return _mantle_config(model) + + def _provider_decline_reason(provider: str, model: str, request: Mapping[str, object]) -> str | None: if provider == "azure": return _azure_decline_reason(model, request) diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py index 65c265030f2..5387b50d26e 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py @@ -5,6 +5,7 @@ recorded OpenAI response, then compares what reached the provider, what the call callback logged. The native leg fails instead of silently falling back to Python """ +import re from collections.abc import Iterator, Mapping from dataclasses import dataclass, field from types import MappingProxyType @@ -33,6 +34,8 @@ _COMPARED_HEADERS: Final = ( "openai-project", "user-agent", ) +_SIGV4_TIME: Final = re.compile(r"/\d{8}/") +_SIGV4_SIGNATURE: Final = re.compile(r"Signature=[0-9a-f]{64}") _COMPARED_LOG_FIELDS: Final = ( "call_type", "custom_llm_provider", @@ -175,6 +178,20 @@ FIREWORKS_INSTRUCTION_INPUT: Final[list[JsonValue]] = [ {"role": "developer", "content": "Now be verbose"}, ] HOSTED_VLLM_MODEL: Final = "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct" +MANTLE_MODEL: Final = "bedrock_mantle/openai.gpt-5.5" +MANTLE_CODEX_INPUT: Final[list[JsonValue]] = [ + {"type": "additional_tools", "tools": [{"type": "mcp", "server_label": "docs"}, {"type": "image_generation"}]}, + {"type": "context_compaction", "encrypted_content": "gAAAAB0a1b2c3d"}, + {"type": "agent_message", "content": [{"type": "input_text", "text": "Planning"}]}, + {"type": "local_shell_call", "call_id": "call_ls", "action": {"type": "exec", "command": ["ls", "-la"]}}, + {"type": "function_call_output", "call_id": "call_ls", "output": "README.md"}, + {"role": "user", "content": "Continue"}, +] +MANTLE_TOOLS: Final[list[JsonValue]] = [ + dict(WEATHER_TOOL), + {"type": "web_search"}, + {"type": "code_interpreter", "container": {"type": "auto"}}, +] XAI_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType( { @@ -391,6 +408,31 @@ CASES: Final = ( "max_output_tokens": 4, }, ), + Case("bedrock_mantle_openai_path", {"model": MANTLE_MODEL, "input": "hi"}), + Case( + "bedrock_mantle_regional_v1_path", + {"model": "bedrock_mantle/us-west-2/openai.gpt-oss-120b", "input": "hi", "aws_bedrock_project_id": "proj_1"}, + ), + Case( + "bedrock_mantle_codex_history_and_tools", + {"model": MANTLE_MODEL, "input": MANTLE_CODEX_INPUT, "tools": MANTLE_TOOLS}, + ), + Case( + "bedrock_mantle_sigv4", + { + "model": MANTLE_MODEL, + "input": "hi", + "api_key": None, + "aws_region_name": "us-west-2", + "aws_access_key_id": "AKIDEXAMPLE", + "aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY", + }, + ), + Case( + "bedrock_mantle_service_tier_stays_on_python", + {"model": MANTLE_MODEL, "input": "hi", "service_tier": "flex", "drop_params": True}, + falls_back=True, + ), Case("azure_o_series_stays_on_python", {"model": "azure/o_series/my-o3", "input": "hi"}, falls_back=True), Case( "tool_schema_repair_stays_on_python", @@ -407,6 +449,11 @@ def _kwargs(case: Case, server: RecordingServer, recorder: RecordingLogger) -> d return {"model": MODEL, "api_key": "sk-test", "api_base": server.base_url, "callbacks": [recorder], **case.kwargs} +def _comparable_header(value: str) -> str: + """A SigV4 signature covers the signing time, so only its scope and signed headers compare""" + return _SIGV4_TIME.sub("//", _SIGV4_SIGNATURE.sub("Signature=", value)) + + def _response(result: object) -> Mapping[str, JsonValue]: assert isinstance(result, ResponsesAPIResponse) return _OBJECT.validate_python(result.model_dump(mode="json", exclude_none=True)) @@ -442,7 +489,7 @@ async def _isolated_leg(case: Case, server: RecordingServer, probe: FallbackProb request_path=sent.path, request_body=_OBJECT.validate_python(sent.body), request_headers=MappingProxyType( - {name: value for name, value in sent.headers.items() if name in _COMPARED_HEADERS} + {name: _comparable_header(value) for name, value in sent.headers.items() if name in _COMPARED_HEADERS} ), response=_response(result), logged=MappingProxyType({name: standard.get(name) for name in _COMPARED_LOG_FIELDS}), diff --git a/tests/unit/rust_bridge/host/routes/test_responses.py b/tests/unit/rust_bridge/host/routes/test_responses.py index b0c2d83b03a..a62e5b6a172 100644 --- a/tests/unit/rust_bridge/host/routes/test_responses.py +++ b/tests/unit/rust_bridge/host/routes/test_responses.py @@ -10,6 +10,7 @@ from litellm.rust_bridge.host.routes.responses import ( decline_reason, map_failure, provider_connection, + provider_params, response, ) from litellm.types.llms.openai import ResponsesAPIResponse @@ -316,3 +317,81 @@ def test_databricks_connection_defaults_ignore_the_global_settings(monkeypatch: monkeypatch.setattr(litellm, "api_key", "global") monkeypatch.setattr(litellm, "api_base", "https://global.invalid") assert connection_defaults("databricks") == (None, None) + + +_MANTLE_MODEL_COST: Final = { + "bedrock_mantle/openai.gpt-openai-path": {"mode": "responses", "use_openai_responses_path": True}, + "bedrock_mantle/openai.gpt-v1-path": {"mode": "chat", "supported_endpoints": ["/v1/responses"]}, + "bedrock_mantle/openai.gpt-chat-only": {"mode": "chat"}, +} + + +@pytest.mark.parametrize( + ("model", "expected"), + [ + pytest.param("bedrock_mantle/openai.gpt-openai-path", (("use_openai_responses_path", True),), id="openai"), + pytest.param( + "bedrock_mantle/us-west-2/openai.gpt-v1-path", (("use_openai_responses_path", False),), id="regional_v1" + ), + pytest.param("openai/gpt-openai-path", (), id="other_provider"), + ], +) +def test_provider_params_carry_the_mantle_path_from_the_cost_map( + monkeypatch: pytest.MonkeyPatch, model: str, expected: tuple[tuple[str, object], ...] +) -> None: + monkeypatch.setattr(litellm, "model_cost", _MANTLE_MODEL_COST) + assert provider_params(model, None) == expected + + +@pytest.mark.parametrize( + ("request_fields", "expected"), + [ + pytest.param({}, None, id="responses_model"), + pytest.param({"model": "bedrock_mantle/openai.gpt-v1-path"}, None, id="supported_endpoint"), + pytest.param( + {"aws_region_name": "us-west-2", "aws_role_name": "arn:aws:iam::1:role/r", "aws_bedrock_project_id": "p"}, + None, + id="aws_settings", + ), + pytest.param( + {"aws_session_tags": {"team": "a"}}, + "native inference does not implement aws_session_tags", + id="aws_session_tags", + ), + pytest.param( + {"model": "bedrock_mantle/openai.gpt-chat-only"}, "native HTTP responses provider", id="chat_only_model" + ), + pytest.param( + {"service_tier": "flex"}, + "native Responses does not implement the model's sampling rules", + id="service_tier", + ), + pytest.param( + {"reasoning": {"summary": "detailed"}}, + "native Responses does not implement the model's sampling rules", + id="openai_path_reasoning_summary", + ), + pytest.param( + {"model": "bedrock_mantle/openai.gpt-v1-path", "reasoning": {"summary": "detailed"}}, + None, + id="v1_path_reasoning_summary", + ), + ], +) +def test_bedrock_mantle_requests_outside_the_native_config_stay_on_python( + monkeypatch: pytest.MonkeyPatch, request_fields: dict[str, object], expected: str | None +) -> None: + monkeypatch.setattr(litellm, "model_cost", _MANTLE_MODEL_COST) + request: Final = {"model": "bedrock_mantle/openai.gpt-openai-path", "input": "hi", **request_fields} + assert decline_reason(request) == expected + + +def test_aws_settings_stay_on_python_for_other_providers() -> None: + request: Final = {"model": "gpt-4.1", "input": "hi", "api_key": "sk-test", "aws_region_name": "us-west-2"} + assert decline_reason(request) == "native inference does not implement aws_region_name" + + +def test_bedrock_mantle_connection_defaults_ignore_the_global_settings(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(litellm, "api_key", "global") + monkeypatch.setattr(litellm, "api_base", "https://global.invalid") + assert connection_defaults("bedrock_mantle") == (None, None)