mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-25 05:54:53 +00:00
Merge 0423ac1164 into 27394d8151
This commit is contained in:
Nate Armstrong
GitHub
commit
0961bba125
25 changed files with 1030 additions and 44 deletions
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -4098,6 +4098,7 @@ dependencies = [
|
|||
"litellm-inference-testing",
|
||||
"litellm-llms",
|
||||
"litellm-llms-types",
|
||||
"litellm-router-types",
|
||||
"litellm-secrets",
|
||||
"litellm-tracing",
|
||||
"reqwest 0.12.28",
|
||||
|
|
|
|||
|
|
@ -532,7 +532,7 @@ pub fn bedrock_model_id_and_region(model: &str) -> (String, Option<String>) {
|
|||
(stripped.to_string(), region)
|
||||
}
|
||||
|
||||
fn is_bedrock_region(value: &str) -> bool {
|
||||
pub fn is_bedrock_region(value: &str) -> bool {
|
||||
value.len() > 3
|
||||
&& value.contains('-')
|
||||
&& value
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ pub(crate) async fn create(
|
|||
api_base: deployment.api_base.clone(),
|
||||
custom_llm_provider: deployment.custom_llm_provider.clone(),
|
||||
api_version: deployment.litellm_params.api_version.clone(),
|
||||
litellm_params: deployment.litellm_params.clone(),
|
||||
extra_headers: None,
|
||||
timeout: deployment.timeout,
|
||||
};
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@ litellm-http.workspace = true
|
|||
litellm-inference.workspace = true
|
||||
litellm-llms.workspace = true
|
||||
litellm-llms-types.workspace = true
|
||||
litellm-router-types.workspace = true
|
||||
litellm-secrets.workspace = true
|
||||
reqwest.workspace = true
|
||||
serde_json = { workspace = true, features = ["preserve_order"] }
|
||||
|
|
|
|||
|
|
@ -5,6 +5,9 @@ use litellm_host::interceptors::{ProviderIdentity, RequestContext};
|
|||
use litellm_llms::{
|
||||
azure::responses::transformation::AZURE_RESPONSES_CONFIG,
|
||||
base_llm::responses::transformation::BaseResponsesApiConfig,
|
||||
bedrock_mantle::responses::transformation::{
|
||||
BedrockMantleResponsesApiConfig, split_region_prefix,
|
||||
},
|
||||
databricks::responses::transformation::DATABRICKS_RESPONSES_CONFIG,
|
||||
fireworks_ai::responses::transformation::FIREWORKS_AI_RESPONSES_CONFIG,
|
||||
hosted_vllm::responses::transformation::HOSTED_VLLM_RESPONSES_CONFIG,
|
||||
|
|
@ -12,6 +15,7 @@ use litellm_llms::{
|
|||
openrouter::responses::transformation::OPENROUTER_RESPONSES_CONFIG,
|
||||
xai::responses::transformation::XAI_RESPONSES_CONFIG,
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
use litellm_secrets::source::SecretSource;
|
||||
use serde_json::Value;
|
||||
|
||||
|
|
@ -25,8 +29,12 @@ pub(super) async fn prepare(
|
|||
call: ResponsesCall,
|
||||
secrets: &dyn SecretSource,
|
||||
) -> Result<ProviderResponsesRequest, Error> {
|
||||
let ResponsesProvider { identity, config } =
|
||||
resolve_provider(&call.model, call.custom_llm_provider.as_deref())?;
|
||||
let ResponsesProvider { identity, config } = resolve_provider(
|
||||
&call.model,
|
||||
call.custom_llm_provider.as_deref(),
|
||||
call.api_base.as_deref(),
|
||||
&call.litellm_params,
|
||||
)?;
|
||||
let provider = identity.provider.as_str();
|
||||
let model = identity.model.as_str();
|
||||
let snapshot = secrets
|
||||
|
|
@ -67,7 +75,7 @@ pub(super) async fn prepare(
|
|||
|
||||
pub(super) struct ResponsesProvider {
|
||||
pub identity: ProviderIdentity,
|
||||
pub config: &'static dyn BaseResponsesApiConfig,
|
||||
pub config: Box<dyn BaseResponsesApiConfig + Send>,
|
||||
}
|
||||
|
||||
/// Python's `get_llm_provider`: a bare model is OpenAI's, and with an explicit OpenRouter
|
||||
|
|
@ -75,6 +83,8 @@ pub(super) struct ResponsesProvider {
|
|||
pub(super) fn resolve_provider(
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
api_base: Option<&str>,
|
||||
params: &LitellmParams,
|
||||
) -> Result<ResponsesProvider, Error> {
|
||||
let explicit = custom_llm_provider.filter(|provider| !provider.is_empty());
|
||||
let (provider, resolved) = match get_custom_llm_provider(model, explicit) {
|
||||
|
|
@ -87,24 +97,34 @@ pub(super) fn resolve_provider(
|
|||
if resolved.is_empty() {
|
||||
return Err(Error::InvalidProvider(model.into()));
|
||||
}
|
||||
let (config, wire_model): (&'static dyn BaseResponsesApiConfig, String) =
|
||||
match provider.parse::<LlmProviders>() {
|
||||
Ok(LlmProviders::Openai) if resolved.contains('/') => {
|
||||
return Err(Error::InvalidProvider(model.into()));
|
||||
}
|
||||
Ok(LlmProviders::Openai) => (&OpenAiResponsesApiConfig, resolved.into()),
|
||||
Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
|
||||
&OPENROUTER_RESPONSES_CONFIG,
|
||||
format!("openrouter/{resolved}"),
|
||||
),
|
||||
Ok(LlmProviders::Openrouter) => (&OPENROUTER_RESPONSES_CONFIG, resolved.into()),
|
||||
Ok(LlmProviders::Azure) => (&AZURE_RESPONSES_CONFIG, resolved.into()),
|
||||
Ok(LlmProviders::Xai) => (&XAI_RESPONSES_CONFIG, resolved.into()),
|
||||
Ok(LlmProviders::Databricks) => (&DATABRICKS_RESPONSES_CONFIG, resolved.into()),
|
||||
Ok(LlmProviders::FireworksAi) => (&FIREWORKS_AI_RESPONSES_CONFIG, resolved.into()),
|
||||
Ok(LlmProviders::HostedVllm) => (&HOSTED_VLLM_RESPONSES_CONFIG, resolved.into()),
|
||||
_ => return Err(Error::Unsupported("native HTTP responses provider")),
|
||||
};
|
||||
let (config, wire_model): (Box<dyn BaseResponsesApiConfig + Send>, String) = match provider
|
||||
.parse::<LlmProviders>(
|
||||
) {
|
||||
Ok(LlmProviders::Openai) if resolved.contains('/') => {
|
||||
return Err(Error::InvalidProvider(model.into()));
|
||||
}
|
||||
Ok(LlmProviders::Openai) => (Box::new(OpenAiResponsesApiConfig), resolved.into()),
|
||||
Ok(LlmProviders::Openrouter) if explicit.is_some() && !resolved.contains('/') => (
|
||||
Box::new(OPENROUTER_RESPONSES_CONFIG),
|
||||
format!("openrouter/{resolved}"),
|
||||
),
|
||||
Ok(LlmProviders::Openrouter) => (Box::new(OPENROUTER_RESPONSES_CONFIG), resolved.into()),
|
||||
Ok(LlmProviders::Azure) => (Box::new(AZURE_RESPONSES_CONFIG), resolved.into()),
|
||||
Ok(LlmProviders::Xai) => (Box::new(XAI_RESPONSES_CONFIG), resolved.into()),
|
||||
Ok(LlmProviders::Databricks) => (Box::new(DATABRICKS_RESPONSES_CONFIG), resolved.into()),
|
||||
Ok(LlmProviders::FireworksAi) => (Box::new(FIREWORKS_AI_RESPONSES_CONFIG), resolved.into()),
|
||||
Ok(LlmProviders::HostedVllm) => (Box::new(HOSTED_VLLM_RESPONSES_CONFIG), resolved.into()),
|
||||
Ok(LlmProviders::BedrockMantle) => {
|
||||
let (region, wire) = split_region_prefix(resolved);
|
||||
(
|
||||
Box::new(BedrockMantleResponsesApiConfig::new(
|
||||
api_base, region, params,
|
||||
)),
|
||||
wire.into(),
|
||||
)
|
||||
}
|
||||
_ => return Err(Error::Unsupported("native HTTP responses provider")),
|
||||
};
|
||||
Ok(ResponsesProvider {
|
||||
identity: ProviderIdentity {
|
||||
model: wire_model,
|
||||
|
|
@ -165,6 +185,12 @@ mod tests {
|
|||
"accounts/fireworks/models/qwen3"
|
||||
)]
|
||||
#[case::xai_prefix("xai/grok-4", None, "xai", "grok-4")]
|
||||
#[case::bedrock_mantle_region_prefix(
|
||||
"bedrock_mantle/us-west-2/openai.gpt-5.5",
|
||||
None,
|
||||
"bedrock_mantle",
|
||||
"openai.gpt-5.5"
|
||||
)]
|
||||
#[case::hosted_vllm_keeps_the_vendor(
|
||||
"hosted_vllm/meta-llama/Llama-3.1-8B",
|
||||
None,
|
||||
|
|
@ -177,7 +203,8 @@ mod tests {
|
|||
#[case] provider: &str,
|
||||
#[case] wire_model: &str,
|
||||
) {
|
||||
let resolved = resolve_provider(model, custom_llm_provider).unwrap();
|
||||
let resolved =
|
||||
resolve_provider(model, custom_llm_provider, None, &LitellmParams::default()).unwrap();
|
||||
assert_eq!(resolved.identity.provider, provider);
|
||||
assert_eq!(resolved.identity.model, wire_model);
|
||||
}
|
||||
|
|
@ -190,7 +217,7 @@ mod tests {
|
|||
#[case::conflicting_prefix("openrouter/anthropic/claude", Some("openai"))]
|
||||
fn everything_else_is_rejected(#[case] model: &str, #[case] custom_llm_provider: Option<&str>) {
|
||||
assert!(matches!(
|
||||
resolve_provider(model, custom_llm_provider),
|
||||
resolve_provider(model, custom_llm_provider, None, &LitellmParams::default()),
|
||||
Err(Error::InvalidProvider(_) | Error::Unsupported(_))
|
||||
));
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ use litellm_llms::base_llm::{
|
|||
auth::ValidatedEnvironment, responses::transformation::BaseResponsesApiConfig,
|
||||
};
|
||||
use litellm_llms_types::formats::responses::ResponsesApiResponse;
|
||||
use litellm_router_types::LitellmParams;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use super::Error;
|
||||
|
|
@ -18,6 +19,8 @@ pub struct ResponsesCall {
|
|||
pub api_base: Option<String>,
|
||||
pub custom_llm_provider: Option<String>,
|
||||
pub api_version: Option<String>,
|
||||
/// The deployment's provider settings, such as AWS credentials and region
|
||||
pub litellm_params: LitellmParams,
|
||||
pub extra_headers: Option<Map<String, Value>>,
|
||||
pub timeout: Option<Duration>,
|
||||
}
|
||||
|
|
@ -29,7 +32,7 @@ pub struct ResponsesStreamHead {
|
|||
pub type ResponsesOutput = CallOutput<ResponsesApiResponse, ResponsesStreamHead, Bytes, Error>;
|
||||
|
||||
pub(super) struct ProviderResponsesRequest {
|
||||
pub config: &'static dyn BaseResponsesApiConfig,
|
||||
pub config: Box<dyn BaseResponsesApiConfig + Send>,
|
||||
pub environment: ValidatedEnvironment,
|
||||
pub url: String,
|
||||
pub body: Value,
|
||||
|
|
|
|||
|
|
@ -285,6 +285,7 @@ async fn responses_cache_identity_follows_resolved_configuration_and_request_cal
|
|||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
api_version: None,
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
timeout: None,
|
||||
},
|
||||
|
|
|
|||
|
|
@ -34,6 +34,7 @@ fn call(stream: bool) -> ResponsesCall {
|
|||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
api_version: None,
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
timeout: Some(Duration::from_secs(60)),
|
||||
}
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ fn call() -> ResponsesCall {
|
|||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
api_version: None,
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
timeout: None,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -28,6 +28,7 @@ fn call() -> ResponsesCall {
|
|||
api_base: None,
|
||||
custom_llm_provider: None,
|
||||
api_version: None,
|
||||
litellm_params: Default::default(),
|
||||
extra_headers: None,
|
||||
timeout: None,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -58,7 +58,7 @@
|
|||
|
||||
- `AnthropicConfig`, `AmazonConverseConfig` and the non-OCR base traits are partial. Preserve their acceptance gates, passthrough and host fallback contracts
|
||||
- `OpenAiResponsesApiConfig` implements WebSocket transformations and direct HTTP Responses, without Python's model-specific param rewriting or Responses-to-Chat emulation
|
||||
- OpenAI-compatible Responses hosts (`openrouter`, `azure`, `xai`, `hosted_vllm`, `fireworks_ai`, `databricks`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules
|
||||
- OpenAI-compatible Responses hosts (`openrouter`, `azure`, `xai`, `hosted_vllm`, `fireworks_ai`, `databricks`, `bedrock_mantle`) reuse the OpenAI request and response transforms as free functions and own only their credentials, URL and host-specific item rules
|
||||
- Chat `supported_openai_param_mappings` lists OpenAI/provider name pairs, unlike Python's name-only list. Audio `map_transcription_params` is a Rust filtering helper
|
||||
- Auth resolution is split between configs and route preparation in `inference-<format>`
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,72 @@
|
|||
//! Tools a client sends as `additional_tools` input items, matching
|
||||
//! `litellm/responses/additional_tools.py`
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
pub struct HoistedAdditionalTools {
|
||||
pub input: Value,
|
||||
pub hoisted: Vec<Value>,
|
||||
}
|
||||
|
||||
/// The input without its `additional_tools` items, and the tools those items carried. An item
|
||||
/// whose `tools` is not a list of objects carries none
|
||||
pub fn hoist_additional_tools(input: Value) -> HoistedAdditionalTools {
|
||||
let Value::Array(items) = input else {
|
||||
return HoistedAdditionalTools {
|
||||
input,
|
||||
hoisted: Vec::new(),
|
||||
};
|
||||
};
|
||||
let (additional, remaining): (Vec<Value>, Vec<Value>) = items
|
||||
.into_iter()
|
||||
.partition(|item| item.get("type").and_then(Value::as_str) == Some("additional_tools"));
|
||||
HoistedAdditionalTools {
|
||||
input: Value::Array(remaining),
|
||||
hoisted: additional.into_iter().flat_map(item_tools).collect(),
|
||||
}
|
||||
}
|
||||
|
||||
fn item_tools(item: Value) -> Vec<Value> {
|
||||
match item.get("tools") {
|
||||
None => Vec::new(),
|
||||
Some(Value::Array(tools)) if tools.iter().all(Value::is_object) => tools.clone(),
|
||||
Some(_) => Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::tools_are_hoisted_in_order(
|
||||
json!([
|
||||
{"type": "additional_tools", "tools": [{"type": "function", "name": "a"}]},
|
||||
{"role": "user", "content": "hi"},
|
||||
{"type": "additional_tools", "tools": [{"type": "mcp", "server_label": "b"}]},
|
||||
]),
|
||||
json!([{"role": "user", "content": "hi"}]),
|
||||
vec![json!({"type": "function", "name": "a"}), json!({"type": "mcp", "server_label": "b"})]
|
||||
)]
|
||||
#[case::malformed_tools_carry_nothing_but_the_item_still_leaves(
|
||||
json!([
|
||||
{"type": "additional_tools", "tools": [{"type": "function"}, "bad"]},
|
||||
{"type": "additional_tools", "tools": null},
|
||||
{"type": "additional_tools"},
|
||||
]),
|
||||
json!([]),
|
||||
vec![]
|
||||
)]
|
||||
#[case::string_input(json!("hi"), json!("hi"), vec![])]
|
||||
fn additional_tools_items_are_hoisted(
|
||||
#[case] input: Value,
|
||||
#[case] remaining: Value,
|
||||
#[case] hoisted: Vec<Value>,
|
||||
) {
|
||||
let result = hoist_additional_tools(input);
|
||||
assert_eq!((result.input, result.hoisted), (remaining, hoisted));
|
||||
}
|
||||
}
|
||||
152
litellm-rust/crates/llms/src/base_llm/responses/codex_compat.rs
Normal file
152
litellm-rust/crates/llms/src/base_llm/responses/codex_compat.rs
Normal file
|
|
@ -0,0 +1,152 @@
|
|||
//! Codex CLI wire-format quirks shared by the Responses providers that need them, matching
|
||||
//! `litellm/llms/base_llm/responses/codex_compat.py`
|
||||
|
||||
use litellm_python_compat::{
|
||||
json::{dumps, from_json},
|
||||
repr::to_str,
|
||||
truthy::truthy,
|
||||
};
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
/// Codex history item types that api.openai.com accepts and other Responses backends reject,
|
||||
/// rewritten to the spelling those backends take. An item that carries nothing to rewrite is
|
||||
/// dropped
|
||||
pub fn normalize_codex_input_items(input: Value) -> Value {
|
||||
match input {
|
||||
Value::Array(items) => Value::Array(items.into_iter().filter_map(normalized).collect()),
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
/// The tools whose `type` the backend accepts; a tool that is not an object passes through
|
||||
pub fn drop_unsupported_tools(tools: Vec<Value>, supported: &[&str]) -> Vec<Value> {
|
||||
tools
|
||||
.into_iter()
|
||||
.filter(|tool| match tool {
|
||||
Value::Object(fields) => fields
|
||||
.get("type")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|kind| supported.contains(&kind)),
|
||||
_ => true,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn normalized(item: Value) -> Option<Value> {
|
||||
let Value::Object(fields) = &item else {
|
||||
return Some(item);
|
||||
};
|
||||
match fields.get("type").and_then(Value::as_str) {
|
||||
Some("agent_message") => agent_message(fields),
|
||||
Some("context_compaction") => context_compaction(fields),
|
||||
Some("local_shell_call") => local_shell_call(fields),
|
||||
_ => Some(item),
|
||||
}
|
||||
}
|
||||
|
||||
fn agent_message(fields: &Map<String, Value>) -> Option<Value> {
|
||||
let text: String = fields
|
||||
.get("content")
|
||||
.and_then(Value::as_array)
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.filter_map(Value::as_object)
|
||||
.map(|block| {
|
||||
["text", "encrypted_content"]
|
||||
.iter()
|
||||
.filter_map(|name| block.get(*name))
|
||||
.map(|value| from_json(value.clone()))
|
||||
.find(truthy)
|
||||
.map(|value| to_str(&value))
|
||||
.unwrap_or_default()
|
||||
})
|
||||
.collect();
|
||||
(!text.is_empty()).then(|| {
|
||||
json!({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": text}]})
|
||||
})
|
||||
}
|
||||
|
||||
fn context_compaction(fields: &Map<String, Value>) -> Option<Value> {
|
||||
fields
|
||||
.get("encrypted_content")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|content| !content.is_empty())
|
||||
.map(|content| json!({"type": "compaction", "encrypted_content": content}))
|
||||
}
|
||||
|
||||
fn local_shell_call(fields: &Map<String, Value>) -> Option<Value> {
|
||||
let call_id = fields
|
||||
.get("call_id")
|
||||
.and_then(Value::as_str)
|
||||
.filter(|call_id| !call_id.is_empty())?;
|
||||
let arguments = match fields.get("action") {
|
||||
Some(action @ Value::Object(_)) => dumps(&from_json(action.clone())).ok()?,
|
||||
_ => "{}".to_owned(),
|
||||
};
|
||||
Some(json!({
|
||||
"type": "function_call",
|
||||
"call_id": call_id,
|
||||
"name": "local_shell",
|
||||
"arguments": arguments,
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[rstest]
|
||||
#[case::agent_message_joins_text_and_plaintext_args(
|
||||
json!({"type": "agent_message", "content": [{"text": "a"}, {"encrypted_content": "b"}, {"text": "", "encrypted_content": "c"}, "skipped"]}),
|
||||
Some(json!({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "abc"}]}))
|
||||
)]
|
||||
#[case::agent_message_renders_non_string_text_like_python(
|
||||
json!({"type": "agent_message", "content": [{"text": 1.0}, {"text": true}]}),
|
||||
Some(json!({"type": "message", "role": "assistant", "content": [{"type": "output_text", "text": "1.0True"}]}))
|
||||
)]
|
||||
#[case::empty_agent_message_is_dropped(json!({"type": "agent_message", "content": [{"text": ""}]}), None)]
|
||||
#[case::context_compaction(
|
||||
json!({"type": "context_compaction", "encrypted_content": "enc"}),
|
||||
Some(json!({"type": "compaction", "encrypted_content": "enc"}))
|
||||
)]
|
||||
#[case::empty_context_compaction_is_dropped(json!({"type": "context_compaction", "encrypted_content": ""}), None)]
|
||||
#[case::local_shell_call_dumps_its_action_like_python(
|
||||
json!({"type": "local_shell_call", "call_id": "c1", "action": {"command": ["ls", "é"], "timeout": 1}}),
|
||||
Some(json!({"type": "function_call", "call_id": "c1", "name": "local_shell", "arguments": "{\"command\": [\"ls\", \"\\u00e9\"], \"timeout\": 1}"}))
|
||||
)]
|
||||
#[case::local_shell_call_without_action(
|
||||
json!({"type": "local_shell_call", "call_id": "c1", "action": "ls"}),
|
||||
Some(json!({"type": "function_call", "call_id": "c1", "name": "local_shell", "arguments": "{}"}))
|
||||
)]
|
||||
#[case::local_shell_call_without_call_id_is_dropped(json!({"type": "local_shell_call", "call_id": ""}), None)]
|
||||
#[case::other_items_pass_through(json!({"type": "message", "role": "user", "content": "hi"}), Some(json!({"type": "message", "role": "user", "content": "hi"})))]
|
||||
fn codex_items_take_the_backend_spelling(#[case] item: Value, #[case] expected: Option<Value>) {
|
||||
assert_eq!(
|
||||
normalize_codex_input_items(json!([item])),
|
||||
Value::Array(expected.into_iter().collect())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn string_input_is_untouched() {
|
||||
assert_eq!(normalize_codex_input_items(json!("hi")), json!("hi"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unsupported_tool_types_are_dropped() {
|
||||
assert_eq!(
|
||||
drop_unsupported_tools(
|
||||
vec![
|
||||
json!({"type": "function", "name": "f"}),
|
||||
json!({"type": "web_search"}),
|
||||
json!({"name": "untyped"}),
|
||||
json!("opaque"),
|
||||
],
|
||||
&["function"],
|
||||
),
|
||||
vec![json!({"type": "function", "name": "f"}), json!("opaque")]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,2 +1,4 @@
|
|||
pub mod additional_tools;
|
||||
pub mod codex_compat;
|
||||
pub mod streaming;
|
||||
pub mod transformation;
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
pub mod messages;
|
||||
pub mod responses;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,12 @@
|
|||
# rules
|
||||
|
||||
- Send the OpenAI Responses payload to `/openai/v1/responses` when the host passes `use_openai_responses_path` from the model cost map, else `/v1/responses`
|
||||
- A standard `bedrock-mantle.<region>.api.aws` base is re-pinned to the resolved region, and any other base keeps its path
|
||||
- Authenticate with a bearer from `api_key`, `BEDROCK_MANTLE_API_KEY` or `AWS_BEARER_TOKEN_BEDROCK`, otherwise sign with AWS SigV4 for service `bedrock`
|
||||
- Keep only the tool types Mantle accepts, hoist `additional_tools` input items into `tools`, and rewrite Codex history items
|
||||
- Models that serve only Chat Completions, and `service_tier` or `reasoning.summary` values Mantle rejects, stay on Python
|
||||
- No native WebSocket mode
|
||||
|
||||
# references
|
||||
|
||||
- Python counterpart `litellm/llms/bedrock_mantle/responses/transformation.py`
|
||||
|
|
@ -0,0 +1 @@
|
|||
pub mod transformation;
|
||||
|
|
@ -0,0 +1,504 @@
|
|||
//! Bedrock Mantle Responses, matching `litellm/llms/bedrock_mantle/responses/transformation.py`
|
||||
|
||||
use std::sync::LazyLock;
|
||||
|
||||
use litellm_auth::{AwsParams, CredentialPlacement, SecretValue};
|
||||
use litellm_auth_aws::{AwsCredentialSource, is_bedrock_region};
|
||||
use litellm_http::request::{with_header, without_headers};
|
||||
use litellm_llms_types::formats::responses::ResponsesApiResponse;
|
||||
use litellm_python_compat::truthy::json_truthy;
|
||||
use litellm_router_types::LitellmParams;
|
||||
use serde_json::{Map, Value};
|
||||
|
||||
use crate::{
|
||||
Error,
|
||||
base_llm::{
|
||||
auth::{AuthScheme, ValidatedEnvironment},
|
||||
responses::{
|
||||
additional_tools::hoist_additional_tools,
|
||||
codex_compat::{drop_unsupported_tools, normalize_codex_input_items},
|
||||
transformation::BaseResponsesApiConfig,
|
||||
},
|
||||
},
|
||||
bedrock::common_utils::validated_region,
|
||||
openai::responses::{
|
||||
request::ResponsesHost,
|
||||
transformation::{responses_request, responses_response},
|
||||
},
|
||||
};
|
||||
|
||||
const KEY_ENVS: [&str; 2] = ["BEDROCK_MANTLE_API_KEY", "AWS_BEARER_TOKEN_BEDROCK"];
|
||||
const API_BASE_ENV: &str = "BEDROCK_MANTLE_API_BASE";
|
||||
const REGION_ENVS: [&str; 3] = ["BEDROCK_MANTLE_REGION", "AWS_REGION_NAME", "AWS_REGION"];
|
||||
const DEFAULT_REGION: &str = "us-east-1";
|
||||
const BASE_SUFFIXES: [&str; 5] = [
|
||||
"/openai/v1/responses",
|
||||
"/v1/responses",
|
||||
"/responses",
|
||||
"/openai/v1",
|
||||
"/v1",
|
||||
];
|
||||
/// Per Bedrock Mantle Responses API validation errors
|
||||
const SUPPORTED_TOOL_TYPES: [&str; 6] = [
|
||||
"function",
|
||||
"mcp",
|
||||
"custom",
|
||||
"namespace",
|
||||
"tool_search",
|
||||
"web_search",
|
||||
];
|
||||
/// The price map's `use_openai_responses_path`, which the host reads for the model
|
||||
const OPENAI_PATH_PARAM: &str = "use_openai_responses_path";
|
||||
const PROJECT_PARAM: &str = "aws_bedrock_project_id";
|
||||
|
||||
/// One call's Mantle connection, since the model picks the Responses path
|
||||
pub struct BedrockMantleResponsesApiConfig {
|
||||
api_base: Option<String>,
|
||||
model_region: Option<String>,
|
||||
aws: AwsParams,
|
||||
project_id: Option<String>,
|
||||
openai_path: bool,
|
||||
}
|
||||
|
||||
impl BedrockMantleResponsesApiConfig {
|
||||
pub fn new(api_base: Option<&str>, model_region: Option<&str>, params: &LitellmParams) -> Self {
|
||||
Self {
|
||||
api_base: api_base.map(str::to_owned),
|
||||
model_region: model_region.map(str::to_owned),
|
||||
aws: params.aws.clone(),
|
||||
project_id: params
|
||||
.extra
|
||||
.get(PROJECT_PARAM)
|
||||
.and_then(Value::as_str)
|
||||
.filter(|project| !project.is_empty())
|
||||
.map(str::to_owned),
|
||||
openai_path: params.extra.get(OPENAI_PATH_PARAM) == Some(&Value::Bool(true)),
|
||||
}
|
||||
}
|
||||
|
||||
/// Python's `resolve_mantle_region`; the model's region prefix stands in for the default
|
||||
/// base `get_llm_provider` builds from it
|
||||
fn region(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
if let Some(region) = non_empty(self.aws.aws_region_name.as_deref()) {
|
||||
return validated_region(region.to_owned());
|
||||
}
|
||||
let routed = match configured_base(api_base, lookup) {
|
||||
Some(base) => mantle_host_region(base.trim_end_matches('/')).map(str::to_owned),
|
||||
None => self.model_region.clone(),
|
||||
};
|
||||
validated_region(
|
||||
routed
|
||||
.or_else(|| {
|
||||
REGION_ENVS
|
||||
.iter()
|
||||
.find_map(|name| lookup(name).filter(|value| !value.is_empty()))
|
||||
})
|
||||
.unwrap_or_else(|| DEFAULT_REGION.into()),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/// A leading Bedrock region in a Mantle model id routes the call and is not sent
|
||||
pub fn split_region_prefix(model: &str) -> (Option<&str>, &str) {
|
||||
match model.split_once('/') {
|
||||
Some((region, rest)) if is_bedrock_region(region) => (Some(region), rest),
|
||||
_ => (None, model),
|
||||
}
|
||||
}
|
||||
|
||||
impl BaseResponsesApiConfig for BedrockMantleResponsesApiConfig {
|
||||
fn secret_names(
|
||||
&self,
|
||||
_api_key: Option<&str>,
|
||||
_api_base: Option<&str>,
|
||||
) -> &'static [&'static str] {
|
||||
static NAMES: LazyLock<Box<[&'static str]>> = LazyLock::new(|| {
|
||||
KEY_ENVS
|
||||
.into_iter()
|
||||
.chain([API_BASE_ENV])
|
||||
.chain(REGION_ENVS)
|
||||
.chain(
|
||||
AwsParams::secret_names()
|
||||
.iter()
|
||||
.copied()
|
||||
.filter(|name| !REGION_ENVS.contains(name)),
|
||||
)
|
||||
.collect()
|
||||
});
|
||||
&NAMES
|
||||
}
|
||||
|
||||
fn validate_environment(
|
||||
&self,
|
||||
headers: Vec<(String, String)>,
|
||||
api_key: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<ValidatedEnvironment, Error> {
|
||||
let headers = with_header(headers, "content-type", "application/json".into());
|
||||
let headers = match &self.project_id {
|
||||
Some(project) => with_header(headers, "OpenAI-Project", project.clone()),
|
||||
None => headers,
|
||||
};
|
||||
let bearer = non_empty(api_key).map(str::to_owned).or_else(|| {
|
||||
KEY_ENVS
|
||||
.iter()
|
||||
.find_map(|name| lookup(name).filter(|value| !value.is_empty()))
|
||||
});
|
||||
Ok(match bearer {
|
||||
Some(token) => ValidatedEnvironment {
|
||||
headers,
|
||||
auth: AuthScheme::Credential {
|
||||
placement: CredentialPlacement::Bearer,
|
||||
secret: SecretValue::new(token),
|
||||
},
|
||||
},
|
||||
None => ValidatedEnvironment {
|
||||
headers: without_headers(headers, &["authorization"]),
|
||||
auth: AuthScheme::AwsSigV4 {
|
||||
region: self.region(self.api_base.as_deref(), lookup)?,
|
||||
service: "bedrock",
|
||||
credentials: Box::new(AwsCredentialSource::from_params(&self.aws, lookup)),
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
fn get_complete_url(
|
||||
&self,
|
||||
api_base: Option<&str>,
|
||||
_api_version: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Result<String, Error> {
|
||||
let region = self.region(api_base, lookup)?;
|
||||
let default = format!("https://bedrock-mantle.{region}.api.aws");
|
||||
let configured = configured_base(api_base, lookup).unwrap_or_else(|| default.clone());
|
||||
let trimmed = configured.trim_end_matches('/');
|
||||
let stripped = BASE_SUFFIXES
|
||||
.iter()
|
||||
.find_map(|suffix| trimmed.strip_suffix(suffix))
|
||||
.unwrap_or(trimmed);
|
||||
let base = match mantle_host_region(stripped) {
|
||||
Some(_) => default.as_str(),
|
||||
None => stripped,
|
||||
};
|
||||
let path = match self.openai_path {
|
||||
true => "/openai/v1/responses",
|
||||
false => "/v1/responses",
|
||||
};
|
||||
Ok(format!("{base}{path}"))
|
||||
}
|
||||
|
||||
fn transform_responses_api_request(
|
||||
&self,
|
||||
model: &str,
|
||||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
let params = with_supported_tools(params);
|
||||
let hoisted = hoist_additional_tools(input);
|
||||
let params = match hoisted.hoisted.is_empty() {
|
||||
true => params,
|
||||
false => with_hoisted_tools(params, hoisted.hoisted),
|
||||
};
|
||||
responses_request(
|
||||
model,
|
||||
normalize_codex_input_items(hoisted.input),
|
||||
params,
|
||||
ResponsesHost::OpenAi,
|
||||
)
|
||||
}
|
||||
|
||||
fn transform_response_api_response(&self, body: Value) -> Result<ResponsesApiResponse, Error> {
|
||||
responses_response(body)
|
||||
}
|
||||
}
|
||||
|
||||
fn non_empty(value: Option<&str>) -> Option<&str> {
|
||||
value.filter(|value| !value.is_empty())
|
||||
}
|
||||
|
||||
fn configured_base(
|
||||
api_base: Option<&str>,
|
||||
lookup: &dyn Fn(&str) -> Option<String>,
|
||||
) -> Option<String> {
|
||||
non_empty(api_base)
|
||||
.map(str::to_owned)
|
||||
.or_else(|| lookup(API_BASE_ENV).filter(|base| !base.is_empty()))
|
||||
}
|
||||
|
||||
/// Python's `MANTLE_HOST_RE`: `^https?://bedrock-mantle\.([^/.]+)\.api\.aws(?=/|$)`, ignoring case
|
||||
fn mantle_host_region(base: &str) -> Option<&str> {
|
||||
let lowered = base.to_ascii_lowercase();
|
||||
let scheme = ["https://", "http://"]
|
||||
.into_iter()
|
||||
.find(|scheme| lowered.starts_with(scheme))?;
|
||||
let host = &base[scheme.len()..];
|
||||
let region_start = "bedrock-mantle.".len();
|
||||
if !host
|
||||
.get(..region_start)?
|
||||
.eq_ignore_ascii_case("bedrock-mantle.")
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let after = &host[region_start..];
|
||||
let region_end = after.find(['.', '/']).filter(|end| *end > 0)?;
|
||||
let rest = &after[region_end..];
|
||||
let suffix = ".api.aws";
|
||||
let matched = rest.get(..suffix.len())?.eq_ignore_ascii_case(suffix)
|
||||
&& matches!(rest.as_bytes().get(suffix.len()), None | Some(b'/'));
|
||||
matched.then(|| &after[..region_end])
|
||||
}
|
||||
|
||||
/// Python's `map_openai_params` tool filter: a single tool is listed, and no supported tool
|
||||
/// leaves no `tools`
|
||||
fn with_supported_tools(mut params: Map<String, Value>) -> Map<String, Value> {
|
||||
if !params.get("tools").is_some_and(json_truthy) {
|
||||
return params;
|
||||
}
|
||||
let listed = match params.remove("tools") {
|
||||
Some(Value::Array(tools)) => tools,
|
||||
tool => tool.into_iter().collect(),
|
||||
};
|
||||
with_tools(
|
||||
params,
|
||||
drop_unsupported_tools(listed, &SUPPORTED_TOOL_TYPES),
|
||||
)
|
||||
}
|
||||
|
||||
fn with_hoisted_tools(mut params: Map<String, Value>, hoisted: Vec<Value>) -> Map<String, Value> {
|
||||
let existing = match params.remove("tools") {
|
||||
Some(Value::Array(tools)) => tools,
|
||||
_ => Vec::new(),
|
||||
};
|
||||
let combined = existing.into_iter().chain(hoisted).collect();
|
||||
with_tools(
|
||||
params,
|
||||
drop_unsupported_tools(combined, &SUPPORTED_TOOL_TYPES),
|
||||
)
|
||||
}
|
||||
|
||||
fn with_tools(mut params: Map<String, Value>, tools: Vec<Value>) -> Map<String, Value> {
|
||||
if !tools.is_empty() {
|
||||
params.insert("tools".into(), Value::Array(tools));
|
||||
}
|
||||
params
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use litellm_router_types::LitellmParams;
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn env(pairs: &'static [(&'static str, &'static str)]) -> impl Fn(&str) -> Option<String> {
|
||||
move |name| {
|
||||
pairs
|
||||
.iter()
|
||||
.find(|(key, _)| *key == name)
|
||||
.map(|(_, value)| (*value).to_owned())
|
||||
}
|
||||
}
|
||||
|
||||
fn params(mut fields: Value) -> LitellmParams {
|
||||
fields["model"] = json!("m");
|
||||
serde_json::from_value(fields).unwrap()
|
||||
}
|
||||
|
||||
fn config(
|
||||
api_base: Option<&str>,
|
||||
model_region: Option<&str>,
|
||||
fields: Value,
|
||||
) -> BedrockMantleResponsesApiConfig {
|
||||
BedrockMantleResponsesApiConfig::new(api_base, model_region, ¶ms(fields))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::default_region(None, None, json!({}), &[], "https://bedrock-mantle.us-east-1.api.aws/v1/responses")]
|
||||
#[case::openai_path(None, None, json!({"use_openai_responses_path": true}), &[], "https://bedrock-mantle.us-east-1.api.aws/openai/v1/responses")]
|
||||
#[case::model_prefix_region(None, Some("us-west-2"), json!({}), &[], "https://bedrock-mantle.us-west-2.api.aws/v1/responses")]
|
||||
#[case::region_env(None, None, json!({}), &[("AWS_REGION", "eu-west-1")], "https://bedrock-mantle.eu-west-1.api.aws/v1/responses")]
|
||||
#[case::aws_region_name_repins_a_mantle_host(
|
||||
Some("https://bedrock-mantle.us-east-1.api.aws/openai/v1"),
|
||||
None,
|
||||
json!({"aws_region_name": "us-west-2", "use_openai_responses_path": true}),
|
||||
&[],
|
||||
"https://bedrock-mantle.us-west-2.api.aws/openai/v1/responses"
|
||||
)]
|
||||
#[case::mantle_host_region_beats_the_env(
|
||||
Some("https://Bedrock-Mantle.eu-central-1.api.aws/v1/responses/"),
|
||||
None,
|
||||
json!({}),
|
||||
&[("BEDROCK_MANTLE_REGION", "us-west-2")],
|
||||
"https://bedrock-mantle.eu-central-1.api.aws/v1/responses"
|
||||
)]
|
||||
#[case::custom_proxy_keeps_its_path(
|
||||
Some("https://proxy.test/tenant/v1"),
|
||||
Some("us-west-2"),
|
||||
json!({}),
|
||||
&[],
|
||||
"https://proxy.test/tenant/v1/responses"
|
||||
)]
|
||||
#[case::base_secret(None, None, json!({}), &[("BEDROCK_MANTLE_API_BASE", "https://proxy.test")], "https://proxy.test/v1/responses")]
|
||||
fn url_matches_python(
|
||||
#[case] api_base: Option<&str>,
|
||||
#[case] model_region: Option<&str>,
|
||||
#[case] fields: Value,
|
||||
#[case] secrets: &'static [(&'static str, &'static str)],
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
assert_eq!(
|
||||
config(api_base, model_region, fields)
|
||||
.get_complete_url(api_base, None, &env(secrets))
|
||||
.unwrap(),
|
||||
expected
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::https("https://bedrock-mantle.us-east-1.api.aws", Some("us-east-1"))]
|
||||
#[case::path("http://bedrock-mantle.us-east-1.api.aws/tenant", Some("us-east-1"))]
|
||||
#[case::port("https://bedrock-mantle.us-east-1.api.aws:443", None)]
|
||||
#[case::other_domain("https://bedrock-mantle.us-east-1.api.aws.test", None)]
|
||||
#[case::empty_region("https://bedrock-mantle..api.aws", None)]
|
||||
#[case::proxy("https://proxy.test", None)]
|
||||
fn mantle_hosts_match_pythons_pattern(#[case] base: &str, #[case] expected: Option<&str>) {
|
||||
assert_eq!(mantle_host_region(base), expected);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::regional("us-west-2/openai.gpt-5.5", Some("us-west-2"), "openai.gpt-5.5")]
|
||||
#[case::gov(
|
||||
"us-gov-west-1/openai.gpt-5.4",
|
||||
Some("us-gov-west-1"),
|
||||
"openai.gpt-5.4"
|
||||
)]
|
||||
#[case::bare("openai.gpt-5.5", None, "openai.gpt-5.5")]
|
||||
fn region_prefixes_route_and_are_not_sent(
|
||||
#[case] model: &str,
|
||||
#[case] region: Option<&str>,
|
||||
#[case] wire: &str,
|
||||
) {
|
||||
assert_eq!(split_region_prefix(model), (region, wire));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::explicit(Some("explicit"), &[("BEDROCK_MANTLE_API_KEY", "env")], "explicit")]
|
||||
#[case::mantle_secret(None, &[("BEDROCK_MANTLE_API_KEY", "mantle"), ("AWS_BEARER_TOKEN_BEDROCK", "aws")], "mantle")]
|
||||
#[case::bedrock_secret(Some(""), &[("AWS_BEARER_TOKEN_BEDROCK", "aws")], "aws")]
|
||||
fn bearer_tokens_skip_sigv4(
|
||||
#[case] api_key: Option<&str>,
|
||||
#[case] secrets: &'static [(&'static str, &'static str)],
|
||||
#[case] expected: &str,
|
||||
) {
|
||||
let environment = config(None, None, json!({}))
|
||||
.validate_environment(vec![], api_key, &env(secrets))
|
||||
.unwrap();
|
||||
let AuthScheme::Credential { placement, secret } = environment.auth else {
|
||||
panic!("expected a bearer credential");
|
||||
};
|
||||
assert_eq!(
|
||||
(placement, secret.expose()),
|
||||
(CredentialPlacement::Bearer, expected)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_bearer_the_call_is_sigv4_signed_for_the_url_region() {
|
||||
let environment = config(
|
||||
Some("https://bedrock-mantle.eu-west-1.api.aws/openai/v1"),
|
||||
Some("us-west-2"),
|
||||
json!({"aws_bedrock_project_id": "proj_1"}),
|
||||
)
|
||||
.validate_environment(
|
||||
vec![("Authorization".into(), "Bearer caller".into())],
|
||||
None,
|
||||
&env(&[]),
|
||||
)
|
||||
.unwrap();
|
||||
assert!(matches!(
|
||||
environment.auth,
|
||||
AuthScheme::AwsSigV4 { ref region, service: "bedrock", .. } if region == "eu-west-1"
|
||||
));
|
||||
assert_eq!(
|
||||
environment.headers,
|
||||
vec![
|
||||
("content-type".to_owned(), "application/json".to_owned()),
|
||||
("openai-project".to_owned(), "proj_1".to_owned()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_invalid_region_is_a_request_error() {
|
||||
assert!(matches!(
|
||||
config(None, None, json!({"aws_region_name": "US EAST"})).get_complete_url(
|
||||
None,
|
||||
None,
|
||||
&env(&[])
|
||||
),
|
||||
Err(Error::InvalidRequest(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::unsupported_dropped(
|
||||
json!([{"type": "function", "name": "f"}, {"type": "file_search"}]),
|
||||
json!("hi"),
|
||||
Some(json!([{"type": "function", "name": "f"}]))
|
||||
)]
|
||||
#[case::single_tool_listed(json!({"type": "web_search"}), json!("hi"), Some(json!([{"type": "web_search"}])))]
|
||||
#[case::none_supported(json!([{"type": "image_generation"}]), json!("hi"), None)]
|
||||
#[case::empty_list_kept(json!([]), json!("hi"), Some(json!([])))]
|
||||
#[case::hoisted_after_existing(
|
||||
json!([{"type": "function", "name": "f"}]),
|
||||
json!([{"type": "additional_tools", "tools": [{"type": "mcp", "server_label": "s"}, {"type": "code_interpreter"}]}, {"role": "user", "content": "hi"}]),
|
||||
Some(json!([{"type": "function", "name": "f"}, {"type": "mcp", "server_label": "s"}]))
|
||||
)]
|
||||
#[case::hoisted_into_empty_tools(
|
||||
json!([]),
|
||||
json!([{"type": "additional_tools", "tools": [{"type": "code_interpreter"}]}, {"role": "user", "content": "hi"}]),
|
||||
None
|
||||
)]
|
||||
fn tools_are_filtered_to_what_mantle_accepts(
|
||||
#[case] tools: Value,
|
||||
#[case] input: Value,
|
||||
#[case] expected: Option<Value>,
|
||||
) {
|
||||
let body = config(None, None, json!({}))
|
||||
.transform_responses_api_request(
|
||||
"openai.gpt-5.5",
|
||||
input,
|
||||
Map::from_iter([("tools".into(), tools)]),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(body.get("tools").cloned(), expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn codex_history_is_rewritten_after_hoisting() {
|
||||
let body = config(None, None, json!({}))
|
||||
.transform_responses_api_request(
|
||||
"openai.gpt-5.5",
|
||||
json!([
|
||||
{"type": "additional_tools", "tools": [{"type": "function", "name": "f"}]},
|
||||
{"type": "context_compaction", "encrypted_content": "enc"},
|
||||
{"role": "user", "content": "hi"},
|
||||
]),
|
||||
Map::new(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
(&body["input"], &body["tools"]),
|
||||
(
|
||||
&json!([{"type": "compaction", "encrypted_content": "enc"}, {"role": "user", "content": "hi"}]),
|
||||
&json!([{"type": "function", "name": "f"}])
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
|
||||
use litellm_auth::{CredentialPlacement, SecretValue};
|
||||
use litellm_llms_types::formats::responses::ResponsesApiResponse;
|
||||
use litellm_python_compat::truthy::json_truthy;
|
||||
use serde_json::{Map, Value, json};
|
||||
|
||||
use crate::{
|
||||
|
|
@ -89,7 +90,7 @@ impl BaseResponsesApiConfig for XaiResponsesApiConfig {
|
|||
mut params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
params.remove("metadata");
|
||||
if let Some(tools) = params.get_mut("tools").filter(|tools| python_truthy(tools)) {
|
||||
if let Some(tools) = params.get_mut("tools").filter(|tools| json_truthy(tools)) {
|
||||
let listed = match tools.take() {
|
||||
Value::Array(tools) => tools,
|
||||
tool => vec![tool],
|
||||
|
|
@ -122,17 +123,6 @@ fn resolved(
|
|||
.or_else(|| lookup(name).filter(|value| !value.is_empty()))
|
||||
}
|
||||
|
||||
fn python_truthy(value: &Value) -> bool {
|
||||
match value {
|
||||
Value::Null => false,
|
||||
Value::Bool(value) => *value,
|
||||
Value::Number(number) => number.as_f64() != Some(0.0),
|
||||
Value::String(value) => !value.is_empty(),
|
||||
Value::Array(values) => !values.is_empty(),
|
||||
Value::Object(values) => !values.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Python's `xai_reported_cost_in_usd`: only a non-negative integer tick count is a cost
|
||||
fn reported_cost(ticks: Option<&Value>) -> Option<f64> {
|
||||
ticks?
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
use std::convert::Infallible;
|
||||
|
||||
use crate::routes::inference::InferenceHost;
|
||||
use crate::{marshal::project_optional_fields, routes::inference::InferenceHost};
|
||||
use litellm_auth::AwsParams;
|
||||
use litellm_core_utils::get_llm_provider_logic::get_custom_llm_provider;
|
||||
use litellm_host_python::{InvokeError, PythonBinding, PythonHostCalls, PythonOwned, from_py};
|
||||
use litellm_inference_responses::{
|
||||
|
|
@ -9,7 +10,9 @@ use litellm_inference_responses::{
|
|||
route::Responses,
|
||||
types::ResponsesCall,
|
||||
};
|
||||
use litellm_router_types::LitellmParams;
|
||||
use pyo3::{
|
||||
exceptions::PyValueError,
|
||||
gc::{PyTraverseError, PyVisit},
|
||||
prelude::*,
|
||||
types::PyDict,
|
||||
|
|
@ -53,6 +56,15 @@ pub(super) fn project(
|
|||
.or(raw("base_url")?),
|
||||
))?
|
||||
.extract()?;
|
||||
let litellm_params = litellm_params(
|
||||
py,
|
||||
&call.options.model,
|
||||
call.options.custom_llm_provider.as_deref(),
|
||||
project_optional_fields(
|
||||
AwsParams::fields().chain(["aws_bedrock_project_id"]),
|
||||
|name| host.argument(py, arguments, name),
|
||||
)?,
|
||||
)?;
|
||||
let optional_params = call
|
||||
.params
|
||||
.into_iter()
|
||||
|
|
@ -89,11 +101,36 @@ pub(super) fn project(
|
|||
.argument(py, arguments, "api_version")?
|
||||
.map(|value| value.extract())
|
||||
.transpose()?,
|
||||
litellm_params,
|
||||
extra_headers: call.options.extra_headers,
|
||||
timeout: call.options.timeout,
|
||||
})
|
||||
}
|
||||
|
||||
/// The caller's AWS settings plus what Python reads from the model cost map for the provider
|
||||
fn litellm_params(
|
||||
py: Python<'_>,
|
||||
model: &str,
|
||||
custom_llm_provider: Option<&str>,
|
||||
fields: serde_json::Map<String, serde_json::Value>,
|
||||
) -> PyResult<LitellmParams> {
|
||||
let provider_params: Vec<(String, serde_json::Value)> = from_py(
|
||||
&py.import(super::ROUTE_HOST_MODULE)?
|
||||
.getattr("provider_params")?
|
||||
.call1((model, custom_llm_provider))?,
|
||||
)?;
|
||||
let fields = [(
|
||||
"model".to_owned(),
|
||||
serde_json::Value::String(model.to_owned()),
|
||||
)]
|
||||
.into_iter()
|
||||
.chain(fields)
|
||||
.chain(provider_params)
|
||||
.collect();
|
||||
serde_json::from_value(serde_json::Value::Object(fields))
|
||||
.map_err(|error| PyValueError::new_err(format!("invalid litellm params: {error}")))
|
||||
}
|
||||
|
||||
/// The provider and deployment Python encodes into the ids it returns: the resolved provider and
|
||||
/// `litellm_metadata.model_info.id`
|
||||
fn id_scope(
|
||||
|
|
|
|||
|
|
@ -47,7 +47,9 @@ fn run_responses(py: Python<'_>, call: NativeCall<'_>, asynchronous: bool) -> Py
|
|||
None => !model.contains('/'),
|
||||
Some(resolved) => match resolved.custom_llm_provider {
|
||||
"openai" => !resolved.model.contains('/'),
|
||||
"azure" | "xai" | "hosted_vllm" | "fireworks_ai" | "databricks" => true,
|
||||
"azure" | "xai" | "hosted_vllm" | "fireworks_ai" | "databricks" | "bedrock_mantle" => {
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
},
|
||||
};
|
||||
|
|
|
|||
|
|
@ -16,3 +16,15 @@ pub fn truthy(value: &Value) -> bool {
|
|||
Value::Dict(entries) => !entries.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// [`truthy`] for the value `json.loads` returns, without converting it
|
||||
pub fn json_truthy(value: &serde_json::Value) -> bool {
|
||||
match value {
|
||||
serde_json::Value::Null => false,
|
||||
serde_json::Value::Bool(value) => *value,
|
||||
serde_json::Value::Number(number) => number.as_f64() != Some(0.0),
|
||||
serde_json::Value::String(value) => !value.is_empty(),
|
||||
serde_json::Value::Array(values) => !values.is_empty(),
|
||||
serde_json::Value::Object(values) => !values.is_empty(),
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ from typing import Final, cast # noqa: TID251 # narrows caller json the checke
|
|||
import litellm
|
||||
from litellm import get_llm_provider
|
||||
from litellm.llms.azure.responses.transformation import AzureOpenAIResponsesAPIConfig
|
||||
from litellm.llms.bedrock_mantle.responses.transformation import BedrockMantleResponsesAPIConfig
|
||||
from litellm.llms.databricks.responses.transformation import DatabricksResponsesAPIConfig
|
||||
from litellm.llms.fireworks_ai.common_utils import get_fireworks_session_id
|
||||
from litellm.llms.fireworks_ai.responses.transformation import FireworksAIResponsesAPIConfig
|
||||
|
|
@ -19,9 +20,25 @@ from litellm.rust_bridge.public_call import inference_decline_reason, known_para
|
|||
from litellm.secret_managers.main import get_secret_str
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
from litellm.types.utils import LlmProviders
|
||||
from litellm.utils import ProviderConfigManager
|
||||
|
||||
PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__)
|
||||
_NATIVE_PARAMETERS: Final = (*PARAMETERS, "api_version")
|
||||
_MANTLE_PARAMETERS: Final = (
|
||||
*_NATIVE_PARAMETERS,
|
||||
"aws_access_key_id",
|
||||
"aws_secret_access_key",
|
||||
"aws_session_token",
|
||||
"aws_region_name",
|
||||
"aws_session_name",
|
||||
"aws_profile_name",
|
||||
"aws_role_name",
|
||||
"aws_web_identity_token",
|
||||
"aws_sts_endpoint",
|
||||
"aws_external_id",
|
||||
"aws_bedrock_project_id",
|
||||
)
|
||||
_CONFIGS: Final[Mapping[str, OpenAIResponsesAPIConfig]] = MappingProxyType(
|
||||
{
|
||||
"openai": OpenAIResponsesAPIConfig(),
|
||||
|
|
@ -40,7 +57,7 @@ def body_params(request: Mapping[str, object]) -> Mapping[str, object]:
|
|||
|
||||
|
||||
def connection_defaults(provider: str) -> tuple[str | None, str | None]:
|
||||
if provider == "databricks":
|
||||
if provider in ("databricks", "bedrock_mantle"):
|
||||
return None, None
|
||||
provider_key: Final = {"openai": litellm.openai_key, "azure": litellm.azure_key}.get(provider)
|
||||
return litellm.api_key or provider_key, litellm.api_base
|
||||
|
|
@ -56,6 +73,21 @@ def provider_connection(
|
|||
return provider_key, provider_base
|
||||
|
||||
|
||||
def provider_params(model: str, custom_llm_provider: str | None) -> tuple[tuple[str, object], ...]:
|
||||
"""The provider settings Python reads from the model cost map, as litellm params"""
|
||||
resolved, provider = _resolved_model({"model": model, "custom_llm_provider": custom_llm_provider})
|
||||
if provider != "bedrock_mantle":
|
||||
return ()
|
||||
config: Final = _mantle_config(resolved)
|
||||
return () if config is None else (("use_openai_responses_path", config.use_openai_path),)
|
||||
|
||||
|
||||
def _mantle_config(model: str) -> BedrockMantleResponsesAPIConfig | None:
|
||||
"""The config Python's dispatch picks from the model cost map, or None for chat-only models"""
|
||||
config: Final = ProviderConfigManager.get_provider_responses_api_config(LlmProviders.BEDROCK_MANTLE, model)
|
||||
return config if isinstance(config, BedrockMantleResponsesAPIConfig) else None
|
||||
|
||||
|
||||
def response(value: Mapping[str, object]) -> ResponsesAPIResponse:
|
||||
return ResponsesAPIResponse.model_validate(value)
|
||||
|
||||
|
|
@ -84,16 +116,22 @@ def decline_reason(request: Mapping[str, object]) -> str | None:
|
|||
model, provider = _resolved_model(request)
|
||||
except litellm.exceptions.BadRequestError:
|
||||
return "native Responses could not resolve the provider"
|
||||
config: Final = _CONFIGS.get(provider)
|
||||
config: Final = _responses_config(provider, model)
|
||||
if config is None:
|
||||
return "native HTTP responses provider"
|
||||
return (
|
||||
inference_decline_reason(_NATIVE_PARAMETERS, request)
|
||||
inference_decline_reason(_MANTLE_PARAMETERS if provider == "bedrock_mantle" else _NATIVE_PARAMETERS, request)
|
||||
or _provider_decline_reason(provider, model, request)
|
||||
or _model_rules_decline_reason(config, model, request)
|
||||
)
|
||||
|
||||
|
||||
def _responses_config(provider: str, model: str) -> OpenAIResponsesAPIConfig | None:
|
||||
if provider != "bedrock_mantle":
|
||||
return _CONFIGS.get(provider)
|
||||
return _mantle_config(model)
|
||||
|
||||
|
||||
def _provider_decline_reason(provider: str, model: str, request: Mapping[str, object]) -> str | None:
|
||||
if provider == "azure":
|
||||
return _azure_decline_reason(model, request)
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ recorded OpenAI response, then compares what reached the provider, what the call
|
|||
callback logged. The native leg fails instead of silently falling back to Python
|
||||
"""
|
||||
|
||||
import re
|
||||
from collections.abc import Iterator, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from types import MappingProxyType
|
||||
|
|
@ -33,6 +34,8 @@ _COMPARED_HEADERS: Final = (
|
|||
"openai-project",
|
||||
"user-agent",
|
||||
)
|
||||
_SIGV4_TIME: Final = re.compile(r"/\d{8}/")
|
||||
_SIGV4_SIGNATURE: Final = re.compile(r"Signature=[0-9a-f]{64}")
|
||||
_COMPARED_LOG_FIELDS: Final = (
|
||||
"call_type",
|
||||
"custom_llm_provider",
|
||||
|
|
@ -175,6 +178,20 @@ FIREWORKS_INSTRUCTION_INPUT: Final[list[JsonValue]] = [
|
|||
{"role": "developer", "content": "Now be verbose"},
|
||||
]
|
||||
HOSTED_VLLM_MODEL: Final = "hosted_vllm/meta-llama/Llama-3.1-8B-Instruct"
|
||||
MANTLE_MODEL: Final = "bedrock_mantle/openai.gpt-5.5"
|
||||
MANTLE_CODEX_INPUT: Final[list[JsonValue]] = [
|
||||
{"type": "additional_tools", "tools": [{"type": "mcp", "server_label": "docs"}, {"type": "image_generation"}]},
|
||||
{"type": "context_compaction", "encrypted_content": "gAAAAB0a1b2c3d"},
|
||||
{"type": "agent_message", "content": [{"type": "input_text", "text": "Planning"}]},
|
||||
{"type": "local_shell_call", "call_id": "call_ls", "action": {"type": "exec", "command": ["ls", "-la"]}},
|
||||
{"type": "function_call_output", "call_id": "call_ls", "output": "README.md"},
|
||||
{"role": "user", "content": "Continue"},
|
||||
]
|
||||
MANTLE_TOOLS: Final[list[JsonValue]] = [
|
||||
dict(WEATHER_TOOL),
|
||||
{"type": "web_search"},
|
||||
{"type": "code_interpreter", "container": {"type": "auto"}},
|
||||
]
|
||||
|
||||
XAI_RESPONSE: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{
|
||||
|
|
@ -391,6 +408,31 @@ CASES: Final = (
|
|||
"max_output_tokens": 4,
|
||||
},
|
||||
),
|
||||
Case("bedrock_mantle_openai_path", {"model": MANTLE_MODEL, "input": "hi"}),
|
||||
Case(
|
||||
"bedrock_mantle_regional_v1_path",
|
||||
{"model": "bedrock_mantle/us-west-2/openai.gpt-oss-120b", "input": "hi", "aws_bedrock_project_id": "proj_1"},
|
||||
),
|
||||
Case(
|
||||
"bedrock_mantle_codex_history_and_tools",
|
||||
{"model": MANTLE_MODEL, "input": MANTLE_CODEX_INPUT, "tools": MANTLE_TOOLS},
|
||||
),
|
||||
Case(
|
||||
"bedrock_mantle_sigv4",
|
||||
{
|
||||
"model": MANTLE_MODEL,
|
||||
"input": "hi",
|
||||
"api_key": None,
|
||||
"aws_region_name": "us-west-2",
|
||||
"aws_access_key_id": "AKIDEXAMPLE",
|
||||
"aws_secret_access_key": "wJalrXUtnFEMI/K7MDENG+bPxRfiCYEXAMPLEKEY",
|
||||
},
|
||||
),
|
||||
Case(
|
||||
"bedrock_mantle_service_tier_stays_on_python",
|
||||
{"model": MANTLE_MODEL, "input": "hi", "service_tier": "flex", "drop_params": True},
|
||||
falls_back=True,
|
||||
),
|
||||
Case("azure_o_series_stays_on_python", {"model": "azure/o_series/my-o3", "input": "hi"}, falls_back=True),
|
||||
Case(
|
||||
"tool_schema_repair_stays_on_python",
|
||||
|
|
@ -407,6 +449,11 @@ def _kwargs(case: Case, server: RecordingServer, recorder: RecordingLogger) -> d
|
|||
return {"model": MODEL, "api_key": "sk-test", "api_base": server.base_url, "callbacks": [recorder], **case.kwargs}
|
||||
|
||||
|
||||
def _comparable_header(value: str) -> str:
|
||||
"""A SigV4 signature covers the signing time, so only its scope and signed headers compare"""
|
||||
return _SIGV4_TIME.sub("/<date>/", _SIGV4_SIGNATURE.sub("Signature=<signature>", value))
|
||||
|
||||
|
||||
def _response(result: object) -> Mapping[str, JsonValue]:
|
||||
assert isinstance(result, ResponsesAPIResponse)
|
||||
return _OBJECT.validate_python(result.model_dump(mode="json", exclude_none=True))
|
||||
|
|
@ -442,7 +489,7 @@ async def _isolated_leg(case: Case, server: RecordingServer, probe: FallbackProb
|
|||
request_path=sent.path,
|
||||
request_body=_OBJECT.validate_python(sent.body),
|
||||
request_headers=MappingProxyType(
|
||||
{name: value for name, value in sent.headers.items() if name in _COMPARED_HEADERS}
|
||||
{name: _comparable_header(value) for name, value in sent.headers.items() if name in _COMPARED_HEADERS}
|
||||
),
|
||||
response=_response(result),
|
||||
logged=MappingProxyType({name: standard.get(name) for name in _COMPARED_LOG_FIELDS}),
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ from litellm.rust_bridge.host.routes.responses import (
|
|||
decline_reason,
|
||||
map_failure,
|
||||
provider_connection,
|
||||
provider_params,
|
||||
response,
|
||||
)
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
|
@ -316,3 +317,81 @@ def test_databricks_connection_defaults_ignore_the_global_settings(monkeypatch:
|
|||
monkeypatch.setattr(litellm, "api_key", "global")
|
||||
monkeypatch.setattr(litellm, "api_base", "https://global.invalid")
|
||||
assert connection_defaults("databricks") == (None, None)
|
||||
|
||||
|
||||
_MANTLE_MODEL_COST: Final = {
|
||||
"bedrock_mantle/openai.gpt-openai-path": {"mode": "responses", "use_openai_responses_path": True},
|
||||
"bedrock_mantle/openai.gpt-v1-path": {"mode": "chat", "supported_endpoints": ["/v1/responses"]},
|
||||
"bedrock_mantle/openai.gpt-chat-only": {"mode": "chat"},
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("model", "expected"),
|
||||
[
|
||||
pytest.param("bedrock_mantle/openai.gpt-openai-path", (("use_openai_responses_path", True),), id="openai"),
|
||||
pytest.param(
|
||||
"bedrock_mantle/us-west-2/openai.gpt-v1-path", (("use_openai_responses_path", False),), id="regional_v1"
|
||||
),
|
||||
pytest.param("openai/gpt-openai-path", (), id="other_provider"),
|
||||
],
|
||||
)
|
||||
def test_provider_params_carry_the_mantle_path_from_the_cost_map(
|
||||
monkeypatch: pytest.MonkeyPatch, model: str, expected: tuple[tuple[str, object], ...]
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", _MANTLE_MODEL_COST)
|
||||
assert provider_params(model, None) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_fields", "expected"),
|
||||
[
|
||||
pytest.param({}, None, id="responses_model"),
|
||||
pytest.param({"model": "bedrock_mantle/openai.gpt-v1-path"}, None, id="supported_endpoint"),
|
||||
pytest.param(
|
||||
{"aws_region_name": "us-west-2", "aws_role_name": "arn:aws:iam::1:role/r", "aws_bedrock_project_id": "p"},
|
||||
None,
|
||||
id="aws_settings",
|
||||
),
|
||||
pytest.param(
|
||||
{"aws_session_tags": {"team": "a"}},
|
||||
"native inference does not implement aws_session_tags",
|
||||
id="aws_session_tags",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": "bedrock_mantle/openai.gpt-chat-only"}, "native HTTP responses provider", id="chat_only_model"
|
||||
),
|
||||
pytest.param(
|
||||
{"service_tier": "flex"},
|
||||
"native Responses does not implement the model's sampling rules",
|
||||
id="service_tier",
|
||||
),
|
||||
pytest.param(
|
||||
{"reasoning": {"summary": "detailed"}},
|
||||
"native Responses does not implement the model's sampling rules",
|
||||
id="openai_path_reasoning_summary",
|
||||
),
|
||||
pytest.param(
|
||||
{"model": "bedrock_mantle/openai.gpt-v1-path", "reasoning": {"summary": "detailed"}},
|
||||
None,
|
||||
id="v1_path_reasoning_summary",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_bedrock_mantle_requests_outside_the_native_config_stay_on_python(
|
||||
monkeypatch: pytest.MonkeyPatch, request_fields: dict[str, object], expected: str | None
|
||||
) -> None:
|
||||
monkeypatch.setattr(litellm, "model_cost", _MANTLE_MODEL_COST)
|
||||
request: Final = {"model": "bedrock_mantle/openai.gpt-openai-path", "input": "hi", **request_fields}
|
||||
assert decline_reason(request) == expected
|
||||
|
||||
|
||||
def test_aws_settings_stay_on_python_for_other_providers() -> None:
|
||||
request: Final = {"model": "gpt-4.1", "input": "hi", "api_key": "sk-test", "aws_region_name": "us-west-2"}
|
||||
assert decline_reason(request) == "native inference does not implement aws_region_name"
|
||||
|
||||
|
||||
def test_bedrock_mantle_connection_defaults_ignore_the_global_settings(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setattr(litellm, "api_key", "global")
|
||||
monkeypatch.setattr(litellm, "api_base", "https://global.invalid")
|
||||
assert connection_defaults("bedrock_mantle") == (None, None)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue