mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-29 06:33:28 +00:00
Merge 258f50dfd9 into 492272a051
This commit is contained in:
Nate Armstrong
GitHub
commit
39a5c3b8a2
11 changed files with 558 additions and 17 deletions
|
|
@ -8,6 +8,15 @@ For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of
|
|||
|
||||
Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all`
|
||||
|
||||
## Existing helpers
|
||||
|
||||
Before writing a small helper that reproduces Python behaviour or inspects JSON, look for it where it belongs and reuse it:
|
||||
|
||||
- `litellm-python-compat` for Python semantics over plain data: type names (`Value::type_name`), truthiness (`truthy`), `repr`/`str`, `json.dumps`/`json.loads` (`json::from_json` turns a `serde_json::Value` into a Python `Value`), `ast.literal_eval` and pickle
|
||||
- `litellm-core-utils` (`core_helpers`) for small utilities shared across crates, such as `json_type_name` and `unix_now`
|
||||
|
||||
If a helper you need is missing, add it to the owning crate instead of writing a private copy in a provider module
|
||||
|
||||
## Test placement
|
||||
|
||||
- Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`
|
||||
|
|
|
|||
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -4174,6 +4174,7 @@ dependencies = [
|
|||
"time",
|
||||
"tokio",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ repository.workspace = true
|
|||
test-support = ["litellm-http/test-support"]
|
||||
|
||||
[dependencies]
|
||||
uuid.workspace = true
|
||||
litellm-llms-types.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
|
||||
|
|
|
|||
|
|
@ -60,6 +60,10 @@ pub enum ErrorDetail {
|
|||
value: String,
|
||||
hint: String,
|
||||
},
|
||||
#[error(
|
||||
"Invalid type for 'tools[{index}].parameters': expected an object, but got {actual} instead."
|
||||
)]
|
||||
InvalidToolParameters { index: usize, actual: &'static str },
|
||||
#[error("invalid {subject} on line {line}: {source}")]
|
||||
InvalidLine {
|
||||
subject: &'static str,
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
pub mod request;
|
||||
pub mod transformation;
|
||||
|
|
|
|||
384
litellm-rust/crates/llms/src/openai/responses/request.rs
Normal file
384
litellm-rust/crates/llms/src/openai/responses/request.rs
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
//! Request shaping shared by every host speaking the OpenAI Responses wire format, matching
|
||||
//! `litellm/llms/openai/responses/transformation.py`
|
||||
|
||||
use litellm_python_compat::{json::from_json, truthy::truthy};
|
||||
use serde_json::{Map, Value};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{Error, ErrorDetail};
|
||||
|
||||
/// The Responses API rejects `max_output_tokens` below this for every model
|
||||
pub const MIN_MAX_OUTPUT_TOKENS: i64 = 16;
|
||||
|
||||
const REASONING_ITEM_ID_PREFIX: &str = "rs_";
|
||||
/// Python's `DEFAULT_MAX_RECURSE_DEPTH` for `filter_value_from_dict`
|
||||
const MAX_STRIP_DEPTH: usize = 100;
|
||||
const REASONING_FIELDS_DROPPED_WHEN_NULL: [&str; 3] = ["status", "content", "encrypted_content"];
|
||||
|
||||
/// Whether the host runs OpenAI's own item validator, which rejects reasoning it did not mint
|
||||
/// and tool-call item ids without its prefixes
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ItemValidation {
|
||||
OpenAi,
|
||||
Lenient,
|
||||
}
|
||||
|
||||
pub fn shape(
|
||||
input: Value,
|
||||
mut params: Map<String, Value>,
|
||||
validation: ItemValidation,
|
||||
) -> Result<(Value, Map<String, Value>), Error> {
|
||||
if let Some(max_output_tokens) = params.get_mut("max_output_tokens")
|
||||
&& max_output_tokens
|
||||
.as_i64()
|
||||
.is_some_and(|tokens| (1..MIN_MAX_OUTPUT_TOKENS).contains(&tokens))
|
||||
{
|
||||
*max_output_tokens = MIN_MAX_OUTPUT_TOKENS.into();
|
||||
}
|
||||
if let Some(Value::Array(tools)) = params.get_mut("tools") {
|
||||
tools
|
||||
.iter_mut()
|
||||
.filter_map(Value::as_object_mut)
|
||||
.for_each(|tool| strip_cache_control(tool, 0));
|
||||
tools
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
.try_for_each(|(index, tool)| decode_tool_parameters(index, tool))?;
|
||||
}
|
||||
let Value::Array(items) = input else {
|
||||
return Ok((input, params));
|
||||
};
|
||||
let items = items
|
||||
.into_iter()
|
||||
.filter(|item| validation == ItemValidation::Lenient || !is_minted_reasoning(item))
|
||||
.map(without_null_reasoning_fields)
|
||||
.map(|mut item| {
|
||||
if let Some(fields) = item.as_object_mut() {
|
||||
strip_cache_control(fields, 0);
|
||||
}
|
||||
item
|
||||
})
|
||||
.map(|item| match validation {
|
||||
ItemValidation::OpenAi => without_foreign_tool_call_id(item),
|
||||
ItemValidation::Lenient => item,
|
||||
})
|
||||
.collect();
|
||||
Ok((Value::Array(items), params))
|
||||
}
|
||||
|
||||
/// Anthropic-only `cache_control` marks, which OpenAI rejects as unknown parameters, removed at
|
||||
/// any depth like Python's `filter_value_from_dict`
|
||||
fn strip_cache_control(fields: &mut Map<String, Value>, depth: usize) {
|
||||
if depth > MAX_STRIP_DEPTH {
|
||||
return;
|
||||
}
|
||||
fields.remove("cache_control");
|
||||
for value in fields.values_mut() {
|
||||
match value {
|
||||
Value::Object(nested) => strip_cache_control(nested, depth + 1),
|
||||
Value::Array(items) => items
|
||||
.iter_mut()
|
||||
.filter_map(Value::as_object_mut)
|
||||
.for_each(|nested| strip_cache_control(nested, depth + 1)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_tool_parameters(index: usize, tool: &mut Value) -> Result<(), Error> {
|
||||
let Some(parameters) = tool
|
||||
.as_object_mut()
|
||||
.and_then(|tool| tool.get_mut("parameters"))
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let decoded = match parameters {
|
||||
Value::Null | Value::Object(_) => return Ok(()),
|
||||
Value::String(encoded) => serde_json::from_str::<Value>(encoded)
|
||||
.ok()
|
||||
.filter(Value::is_object),
|
||||
_ => None,
|
||||
};
|
||||
let Some(decoded) = decoded else {
|
||||
return Err(Error::InvalidRequest(ErrorDetail::InvalidToolParameters {
|
||||
index,
|
||||
actual: from_json(parameters.clone()).type_name(),
|
||||
}));
|
||||
};
|
||||
*parameters = decoded;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reasoning LiteLLM minted while bridging another provider, which OpenAI cannot decrypt
|
||||
fn is_minted_reasoning(item: &Value) -> bool {
|
||||
let Some(item) = item.as_object() else {
|
||||
return false;
|
||||
};
|
||||
item.get("type").and_then(Value::as_str) == Some("reasoning")
|
||||
&& (item
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(is_minted_reasoning_item_id)
|
||||
|| item
|
||||
.get("encrypted_content")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(carries_thinking_blocks))
|
||||
}
|
||||
|
||||
fn is_minted_reasoning_item_id(id: &str) -> bool {
|
||||
id.strip_prefix(REASONING_ITEM_ID_PREFIX)
|
||||
.is_some_and(|suffix| {
|
||||
Uuid::try_parse(suffix).is_ok_and(|parsed| {
|
||||
parsed.get_version_num() == 4 && parsed.hyphenated().to_string() == suffix
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn carries_thinking_blocks(encrypted_content: &str) -> bool {
|
||||
let Ok(Value::Array(blocks)) = serde_json::from_str::<Value>(encrypted_content) else {
|
||||
return false;
|
||||
};
|
||||
blocks.iter().filter_map(Value::as_object).any(|block| {
|
||||
let is_set = |field: &str| {
|
||||
block
|
||||
.get(field)
|
||||
.is_some_and(|value| truthy(&from_json(value.clone())))
|
||||
};
|
||||
match block.get("type").and_then(Value::as_str) {
|
||||
Some("thinking") => is_set("signature"),
|
||||
Some("redacted_thinking") => is_set("data"),
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// OpenAI rejects `status: null` on a replayed reasoning item. Python round-trips a valid item
|
||||
/// through the SDK model with `exclude_none`, and otherwise drops only the nullable fields
|
||||
fn without_null_reasoning_fields(item: Value) -> Value {
|
||||
let Value::Object(mut fields) = item else {
|
||||
return item;
|
||||
};
|
||||
if fields.get("type").and_then(Value::as_str) != Some("reasoning") {
|
||||
return Value::Object(fields);
|
||||
}
|
||||
if is_sdk_reasoning_item(&fields) {
|
||||
drop_nulls(&mut fields);
|
||||
fields
|
||||
.iter_mut()
|
||||
.filter(|(name, _)| matches!(name.as_str(), "summary" | "content"))
|
||||
.filter_map(|(_, parts)| parts.as_array_mut())
|
||||
.flatten()
|
||||
.filter_map(Value::as_object_mut)
|
||||
.for_each(drop_nulls);
|
||||
} else {
|
||||
fields.retain(|name, value| {
|
||||
!(value.is_null() && REASONING_FIELDS_DROPPED_WHEN_NULL.contains(&name.as_str()))
|
||||
});
|
||||
}
|
||||
Value::Object(fields)
|
||||
}
|
||||
|
||||
/// Whether `ResponseReasoningItem.model_validate` accepts the item
|
||||
fn is_sdk_reasoning_item(fields: &Map<String, Value>) -> bool {
|
||||
let parts = |name: &str, part_type: &str, required: bool| match fields.get(name) {
|
||||
None | Some(Value::Null) => !required,
|
||||
Some(Value::Array(parts)) => parts.iter().all(|part| {
|
||||
part.get("type").and_then(Value::as_str) == Some(part_type)
|
||||
&& part.get("text").is_some_and(Value::is_string)
|
||||
}),
|
||||
Some(_) => false,
|
||||
};
|
||||
fields.get("id").is_some_and(Value::is_string)
|
||||
&& parts("summary", "summary_text", true)
|
||||
&& parts("content", "reasoning_text", false)
|
||||
&& fields
|
||||
.get("encrypted_content")
|
||||
.is_none_or(|value| value.is_null() || value.is_string())
|
||||
&& fields.get("status").is_none_or(|status| {
|
||||
status.is_null()
|
||||
|| matches!(
|
||||
status.as_str(),
|
||||
Some("in_progress" | "completed" | "incomplete")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn drop_nulls(fields: &mut Map<String, Value>) {
|
||||
fields.retain(|_, value| !value.is_null());
|
||||
}
|
||||
|
||||
fn without_foreign_tool_call_id(item: Value) -> Value {
|
||||
let Value::Object(mut fields) = item else {
|
||||
return item;
|
||||
};
|
||||
let prefix = match fields.get("type").and_then(Value::as_str) {
|
||||
Some("function_call") => "fc",
|
||||
Some("custom_tool_call") => "ctc",
|
||||
_ => return Value::Object(fields),
|
||||
};
|
||||
if fields
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|id| !id.starts_with(prefix))
|
||||
{
|
||||
fields.remove("id");
|
||||
}
|
||||
Value::Object(fields)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn shaped(input: Value, params: Value, validation: ItemValidation) -> (Value, Value) {
|
||||
let Value::Object(params) = params else {
|
||||
panic!("params must be an object")
|
||||
};
|
||||
let (input, params) = shape(input, params, validation).unwrap();
|
||||
(input, Value::Object(params))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::warm_up_probe(json!(1), json!(16))]
|
||||
#[case::just_below(json!(15), json!(16))]
|
||||
#[case::minimum(json!(16), json!(16))]
|
||||
#[case::zero_left_for_the_provider(json!(0), json!(0))]
|
||||
#[case::negative_left_for_the_provider(json!(-1), json!(-1))]
|
||||
#[case::null(Value::Null, Value::Null)]
|
||||
fn raises_max_output_tokens_to_the_minimum(#[case] requested: Value, #[case] sent: Value) {
|
||||
let (_, params) = shaped(
|
||||
json!("hi"),
|
||||
json!({"max_output_tokens": requested}),
|
||||
ItemValidation::Lenient,
|
||||
);
|
||||
assert_eq!(params, json!({"max_output_tokens": sent}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_cache_control_at_any_depth() {
|
||||
let mark = json!({"type": "ephemeral"});
|
||||
let (input, params) = shaped(
|
||||
json!([
|
||||
{"role": "user", "content": "hi", "cache_control": mark},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "a", "cache_control": mark}]},
|
||||
{"type": "function_call_output", "call_id": "c", "output": [{"type": "input_text", "text": "b", "cache_control": mark}]}
|
||||
]),
|
||||
json!({"tools": [{"type": "function", "name": "a", "cache_control": mark, "parameters": {"type": "object"}}]}),
|
||||
ItemValidation::Lenient,
|
||||
);
|
||||
assert_eq!(
|
||||
(input, params),
|
||||
(
|
||||
json!([
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "a"}]},
|
||||
{"type": "function_call_output", "call_id": "c", "output": [{"type": "input_text", "text": "b"}]}
|
||||
]),
|
||||
json!({"tools": [{"type": "function", "name": "a", "parameters": {"type": "object"}}]})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_json_encoded_tool_parameters() {
|
||||
let (_, params) = shaped(
|
||||
json!("hi"),
|
||||
json!({"tools": [
|
||||
{"type": "function", "name": "a", "parameters": "{\"type\":\"object\"}"},
|
||||
{"type": "function", "name": "b", "parameters": {"type": "object"}},
|
||||
{"type": "function", "name": "c", "parameters": null},
|
||||
{"type": "web_search"}
|
||||
]}),
|
||||
ItemValidation::Lenient,
|
||||
);
|
||||
assert_eq!(
|
||||
params,
|
||||
json!({"tools": [
|
||||
{"type": "function", "name": "a", "parameters": {"type": "object"}},
|
||||
{"type": "function", "name": "b", "parameters": {"type": "object"}},
|
||||
{"type": "function", "name": "c", "parameters": null},
|
||||
{"type": "web_search"}
|
||||
]})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::not_json(json!("{"), "str")]
|
||||
#[case::json_array(json!("[]"), "str")]
|
||||
#[case::number(json!(3), "int")]
|
||||
#[case::list(json!([]), "list")]
|
||||
fn rejects_tool_parameters_that_are_not_objects(
|
||||
#[case] parameters: Value,
|
||||
#[case] actual: &str,
|
||||
) {
|
||||
let Value::Object(params) = json!({"tools": [{"type": "web_search"}, {"type": "function", "parameters": parameters}]})
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(
|
||||
shape(json!("hi"), params, ItemValidation::Lenient)
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
format!(
|
||||
"invalid request: Invalid type for 'tools[1].parameters': expected an object, but got {actual} instead."
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::sdk_shaped(
|
||||
json!({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "x"}], "status": null, "encrypted_content": null, "extra": null}),
|
||||
json!({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "x"}]})
|
||||
)]
|
||||
#[case::missing_id_keeps_unknown_nulls(
|
||||
json!({"type": "reasoning", "summary": [], "status": null, "extra": null}),
|
||||
json!({"type": "reasoning", "summary": [], "extra": null})
|
||||
)]
|
||||
#[case::missing_summary(
|
||||
json!({"type": "reasoning", "id": "rs_1", "content": null}),
|
||||
json!({"type": "reasoning", "id": "rs_1"})
|
||||
)]
|
||||
#[case::not_reasoning(
|
||||
json!({"type": "message", "role": "user", "content": "hi", "status": null}),
|
||||
json!({"type": "message", "role": "user", "content": "hi", "status": null})
|
||||
)]
|
||||
fn drops_null_reasoning_fields(#[case] item: Value, #[case] expected: Value) {
|
||||
let (input, _) = shaped(json!([item]), json!({}), ItemValidation::Lenient);
|
||||
assert_eq!(input, json!([expected]));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::minted_id(json!({"type": "reasoning", "id": "rs_6f9619ff-8b86-4d01-b42d-00c04fc964ff", "summary": []}), true)]
|
||||
#[case::bridged_thinking(
|
||||
json!({"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "[{\"type\":\"thinking\",\"thinking\":\"x\",\"signature\":\"sig\"}]"}),
|
||||
true
|
||||
)]
|
||||
#[case::unsigned_thinking(
|
||||
json!({"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "[{\"type\":\"thinking\",\"thinking\":\"x\"}]"}),
|
||||
false
|
||||
)]
|
||||
#[case::openai_id(json!({"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "encrypted_content": "gAAAA"}), false)]
|
||||
#[case::uppercase_uuid(json!({"type": "reasoning", "id": "rs_6F9619FF-8B86-4D01-B42D-00C04FC964FF", "summary": []}), false)]
|
||||
#[case::uuid_v1(json!({"type": "reasoning", "id": "rs_6f9619ff-8b86-11d0-b42d-00c04fc964ff", "summary": []}), false)]
|
||||
fn drops_minted_reasoning_for_openai(#[case] item: Value, #[case] minted: bool) {
|
||||
let input = json!([item, {"role": "user", "content": "hi"}]);
|
||||
let (openai, _) = shaped(input.clone(), json!({}), ItemValidation::OpenAi);
|
||||
let (lenient, _) = shaped(input.clone(), json!({}), ItemValidation::Lenient);
|
||||
assert_eq!(openai.as_array().unwrap().len(), if minted { 1 } else { 2 });
|
||||
assert_eq!(lenient, input);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::foreign_function_call(json!({"type": "function_call", "id": "toolu_1", "call_id": "c"}), json!({"type": "function_call", "call_id": "c"}))]
|
||||
#[case::openai_function_call(json!({"type": "function_call", "id": "fc_1", "call_id": "c"}), json!({"type": "function_call", "id": "fc_1", "call_id": "c"}))]
|
||||
#[case::foreign_custom_tool_call(json!({"type": "custom_tool_call", "id": "x", "call_id": "c"}), json!({"type": "custom_tool_call", "call_id": "c"}))]
|
||||
#[case::other_item(json!({"type": "message", "id": "x", "role": "user", "content": "hi"}), json!({"type": "message", "id": "x", "role": "user", "content": "hi"}))]
|
||||
fn drops_foreign_tool_call_ids_for_openai(#[case] item: Value, #[case] expected: Value) {
|
||||
let (openai, _) = shaped(json!([item.clone()]), json!({}), ItemValidation::OpenAi);
|
||||
let (lenient, _) = shaped(json!([item.clone()]), json!({}), ItemValidation::Lenient);
|
||||
assert_eq!((openai, lenient), (json!([expected]), json!([item])));
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ use serde_json::{Map, Value};
|
|||
|
||||
use litellm_auth::{CredentialPlacement, SecretValue};
|
||||
|
||||
use super::request::{self, ItemValidation};
|
||||
use crate::{
|
||||
Error,
|
||||
base_llm::{
|
||||
|
|
@ -165,7 +166,7 @@ impl BaseResponsesApiConfig for OpenAiResponsesApiConfig {
|
|||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
responses_request(model, input, params)
|
||||
responses_request(model, input, params, ItemValidation::OpenAi)
|
||||
}
|
||||
|
||||
fn transform_response_api_response(&self, body: Value) -> Result<ResponsesApiResponse, Error> {
|
||||
|
|
@ -178,6 +179,7 @@ pub fn responses_request(
|
|||
model: &str,
|
||||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
validation: ItemValidation,
|
||||
) -> Result<Value, Error> {
|
||||
if !input.is_string() && !input.is_array() {
|
||||
return Err(Error::InvalidRequest(
|
||||
|
|
@ -190,6 +192,7 @@ pub fn responses_request(
|
|||
{
|
||||
return Err(Error::InvalidRequest("stream must be a boolean".into()));
|
||||
}
|
||||
let (input, params) = request::shape(input, params, validation)?;
|
||||
Ok(Value::Object(
|
||||
params
|
||||
.into_iter()
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ use crate::{
|
|||
auth::{AuthScheme, ValidatedEnvironment},
|
||||
responses::transformation::BaseResponsesApiConfig,
|
||||
},
|
||||
openai::responses::transformation::{responses_request, responses_response},
|
||||
openai::responses::{
|
||||
request::ItemValidation,
|
||||
transformation::{responses_request, responses_response},
|
||||
},
|
||||
};
|
||||
|
||||
const API_KEY_ENVS: [&str; 2] = ["OPENROUTER_API_KEY", "OR_API_KEY"];
|
||||
|
|
@ -77,7 +80,7 @@ impl BaseResponsesApiConfig for OpenRouterResponsesApiConfig {
|
|||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
responses_request(model, input, params)
|
||||
responses_request(model, input, params, ItemValidation::Lenient)
|
||||
}
|
||||
|
||||
fn transform_response_api_response(&self, body: Value) -> Result<ResponsesApiResponse, Error> {
|
||||
|
|
|
|||
|
|
@ -1,15 +1,18 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from typing import Final, cast # noqa: TID251 # narrows caller json the checker cannot see into
|
||||
|
||||
import litellm
|
||||
from litellm import get_llm_provider
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.rust_bridge import failures
|
||||
from litellm.rust_bridge.public_call import inference_decline_reason, known_params, optional_str
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__)
|
||||
_OPENAI_CONFIG: Final = OpenAIResponsesAPIConfig()
|
||||
|
||||
|
||||
def body_params(request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -44,4 +47,52 @@ def decline_reason(request: Mapping[str, object]) -> str | None:
|
|||
return "native Responses could not resolve the provider"
|
||||
if provider != "openai":
|
||||
return "native HTTP responses provider"
|
||||
return inference_decline_reason(PARAMETERS, request)
|
||||
return inference_decline_reason(PARAMETERS, request) or _model_rules_decline_reason(request)
|
||||
|
||||
|
||||
def _model_rules_decline_reason(request: Mapping[str, object]) -> str | None:
|
||||
"""Request rewrites that read the model cost map or Python's regex engine stay on Python"""
|
||||
model: Final = str(request["model"]).removeprefix("openai/")
|
||||
params: Final = {name: request[name] for name in PARAMETERS if request.get(name) is not None}
|
||||
try:
|
||||
_OPENAI_CONFIG.map_openai_params(params, model, drop_params=False) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # the same rules the Python route applies
|
||||
except litellm.UnsupportedParamsError:
|
||||
return "native Responses does not implement the model's sampling rules"
|
||||
litellm_params: Final = GenericLiteLLMParams(
|
||||
api_base=optional_str(request.get("api_base")) or optional_str(request.get("base_url")),
|
||||
model_info=request.get("model_info"),
|
||||
)
|
||||
if _repairs_tool_schemas(model, request.get("tools"), litellm_params):
|
||||
return "native Responses does not implement tool schema repair"
|
||||
if _carries_cache_control(request.get("input")) and _OPENAI_CONFIG._reads_prompt_cache_breakpoints( # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies
|
||||
model=model, litellm_params=litellm_params, prompt_cache_options=request.get("prompt_cache_options")
|
||||
):
|
||||
return "native Responses does not implement prompt cache breakpoints"
|
||||
return None
|
||||
|
||||
|
||||
def _repairs_tool_schemas(model: str, tools: object, litellm_params: GenericLiteLLMParams) -> bool:
|
||||
if not isinstance(tools, list):
|
||||
return False
|
||||
try:
|
||||
decoded: Final = _OPENAI_CONFIG._tools_with_object_parameters(model=model, tools=cast(list[object], tools)) # pyright: ignore[reportPrivateUsage, reportArgumentType] # cast-ok: caller json
|
||||
except litellm.BadRequestError:
|
||||
return False
|
||||
return (
|
||||
_OPENAI_CONFIG._sanitized_tool_schemas_for_openai(model=model, tools=decoded, litellm_params=litellm_params) # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies
|
||||
is not decoded
|
||||
)
|
||||
|
||||
|
||||
def _carries_cache_control(value: object) -> bool:
|
||||
pending: Final[list[object]] = [value] # mutable-ok: work stack for a depth-first walk of caller json
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
if isinstance(current, Mapping):
|
||||
fields = cast(Mapping[str, object], current) # cast-ok: caller json
|
||||
if "cache_control" in fields:
|
||||
return True
|
||||
pending.extend(fields.values())
|
||||
elif isinstance(current, list):
|
||||
pending.extend(cast(list[object], current)) # cast-ok: caller json
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -156,6 +156,32 @@ AFFINITY_REPLAY_INPUT: Final[list[JsonValue]] = [
|
|||
{"role": "user", "content": "And Germany?"},
|
||||
]
|
||||
|
||||
MINTED_REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [
|
||||
{"type": "reasoning", "id": "rs_6f9619ff-8b86-4d01-b42d-00c04fc964ff", "summary": []},
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_0a1b2c3d4e5f",
|
||||
"summary": [],
|
||||
"encrypted_content": '[{"type":"thinking","thinking":"x","signature":"sig"}]',
|
||||
},
|
||||
{"role": "user", "content": "And Germany?"},
|
||||
]
|
||||
|
||||
FOREIGN_TOOL_CALL_REPLAY_INPUT: Final[list[JsonValue]] = [
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "toolu_6f5e4d3c2b1a",
|
||||
"call_id": "call_6f5e4d3c2b1a",
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city":"Paris"}',
|
||||
},
|
||||
{"type": "function_call_output", "call_id": "call_6f5e4d3c2b1a", "output": "18C and sunny"},
|
||||
]
|
||||
|
||||
PATTERNED_PARAMETERS: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{"type": "object", "properties": {"city": {"type": "string", "pattern": "^\\p{L}+$"}}}
|
||||
)
|
||||
|
||||
REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [
|
||||
{"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "status": None},
|
||||
{"role": "user", "content": "And Germany?"},
|
||||
|
|
@ -183,6 +209,7 @@ class Case:
|
|||
kwargs: Mapping[str, object]
|
||||
upstream: Mapping[str, JsonValue] = TEXT_RESPONSE
|
||||
gaps: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({}))
|
||||
falls_back: bool = False
|
||||
|
||||
def params(self) -> Iterator[object]:
|
||||
for aspect in ASPECTS:
|
||||
|
|
@ -196,10 +223,6 @@ DEPLOYMENT_METADATA: Final = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
def _gaps(**gaps: str) -> Mapping[str, str]:
|
||||
return MappingProxyType(gaps)
|
||||
|
||||
|
||||
CASES: Final = (
|
||||
Case("plain_text", {"input": "What is the capital of France?"}),
|
||||
Case(
|
||||
|
|
@ -236,14 +259,11 @@ CASES: Final = (
|
|||
Case(
|
||||
"warm_up_probe_below_minimum_max_output_tokens",
|
||||
{"input": "hi", "max_output_tokens": 1},
|
||||
gaps=_gaps(request="native requests send max_output_tokens below OpenAI's minimum of 16"),
|
||||
),
|
||||
Case(
|
||||
"gpt_5_family_drops_temperature_while_reasoning",
|
||||
{"input": "hi", "temperature": 0.2, "reasoning": {"effort": "high"}, "drop_params": True},
|
||||
gaps=MappingProxyType(
|
||||
{"dispatch": "native responses decline drop_params instead of applying the GPT-5 sampling rules"}
|
||||
),
|
||||
falls_back=True,
|
||||
),
|
||||
Case(
|
||||
"anthropic_cache_control_is_stripped",
|
||||
|
|
@ -262,7 +282,6 @@ CASES: Final = (
|
|||
{
|
||||
"input": REASONING_REPLAY_INPUT,
|
||||
},
|
||||
gaps=_gaps(request="native requests forward reasoning items with status null"),
|
||||
),
|
||||
Case(
|
||||
"tool_parameters_sent_as_json_string",
|
||||
|
|
@ -270,7 +289,26 @@ CASES: Final = (
|
|||
"input": "Weather in Paris?",
|
||||
"tools": [{**WEATHER_TOOL, "parameters": '{"type":"object","properties":{"city":{"type":"string"}}}'}],
|
||||
},
|
||||
gaps=_gaps(request="native requests forward tool parameters encoded as a JSON string"),
|
||||
),
|
||||
Case(
|
||||
"item_level_cache_control_is_stripped",
|
||||
{"input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}]},
|
||||
),
|
||||
Case(
|
||||
"bridge_minted_reasoning_is_dropped",
|
||||
{"input": MINTED_REASONING_REPLAY_INPUT},
|
||||
),
|
||||
Case(
|
||||
"foreign_tool_call_ids_are_dropped",
|
||||
{"input": FOREIGN_TOOL_CALL_REPLAY_INPUT, "tools": [dict(WEATHER_TOOL)]},
|
||||
),
|
||||
Case(
|
||||
"tool_schema_repair_stays_on_python",
|
||||
{
|
||||
"input": "Weather in Paris?",
|
||||
"tools": [{**WEATHER_TOOL, "parameters": dict(PATTERNED_PARAMETERS)}],
|
||||
},
|
||||
falls_back=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -351,7 +389,7 @@ async def test_native_responses_match_python(
|
|||
native: Final = await _leg(case, parity_server, probe=native_responses)
|
||||
match aspect:
|
||||
case "dispatch":
|
||||
assert not native.fell_back
|
||||
assert native.fell_back == case.falls_back
|
||||
case "request":
|
||||
assert (native.request_body, native.request_headers) == (python.request_body, python.request_headers)
|
||||
case "response":
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import pytest
|
|||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.rust_bridge.host.routes.responses import connection_defaults, map_failure, response
|
||||
from litellm.rust_bridge.host.routes.responses import connection_defaults, decline_reason, map_failure, response
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
|
||||
|
|
@ -87,3 +87,49 @@ def test_failure_preserves_the_explicit_endpoint(api_base: str | None, base_url:
|
|||
assert isinstance(mapped, litellm.RateLimitError)
|
||||
assert str(mapped.response.request.url) == expected
|
||||
assert mapped.__context__ is upstream
|
||||
|
||||
|
||||
_PATTERNED_TOOL: Final = MappingProxyType(
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string", "pattern": "^\\p{L}+$"}}},
|
||||
}
|
||||
)
|
||||
_PLAIN_TOOL: Final = MappingProxyType(
|
||||
{"type": "function", "name": "lookup", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_fields", "expected"),
|
||||
(
|
||||
pytest.param({"input": "hi", "temperature": 1}, None, id="supported_temperature"),
|
||||
pytest.param(
|
||||
{"input": "hi", "temperature": 0.2},
|
||||
"native Responses does not implement the model's sampling rules",
|
||||
id="gpt_5_sampling_rules",
|
||||
),
|
||||
pytest.param({"input": "hi", "tools": [dict(_PLAIN_TOOL)]}, None, id="valid_tool_schema"),
|
||||
pytest.param(
|
||||
{"input": "hi", "tools": [dict(_PATTERNED_TOOL)]},
|
||||
"native Responses does not implement tool schema repair",
|
||||
id="python_incompatible_pattern",
|
||||
),
|
||||
pytest.param(
|
||||
{"input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}]},
|
||||
"native Responses does not implement prompt cache breakpoints",
|
||||
id="prompt_cache_breakpoints_on_openai",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}],
|
||||
"api_base": "http://127.0.0.1:9/v1",
|
||||
},
|
||||
None,
|
||||
id="cache_control_on_a_compatible_host",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_model_rules_stay_on_python(request_fields: dict[str, object], expected: str | None) -> None:
|
||||
assert decline_reason(MappingProxyType({"model": "openai/gpt-6.1-sol", **request_fields})) == expected
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue