mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-26 06:03:19 +00:00
feat(rust): shape native OpenAI responses requests like Python
The OpenAI Responses transformation now raises max_output_tokens to the minimum of 16, strips Anthropic cache_control marks, decodes tool parameters sent as JSON strings and drops null fields from replayed reasoning items. Against OpenAI it also drops reasoning LiteLLM minted while bridging another provider and tool-call item ids OpenAI did not issue. Rewrites that read the model cost map or Python's regex engine (GPT-5 sampling rules, tool schema repair, prompt cache breakpoints) decline to the Python route
This commit is contained in:
narmstrong-berry
narmstrong-berri
parent
492272a051
commit
258f50dfd9
11 changed files with 558 additions and 17 deletions
|
|
@ -8,6 +8,15 @@ For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of
|
|||
|
||||
Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all`
|
||||
|
||||
## Existing helpers
|
||||
|
||||
Before writing a small helper that reproduces Python behaviour or inspects JSON, look for it where it belongs and reuse it:
|
||||
|
||||
- `litellm-python-compat` for Python semantics over plain data: type names (`Value::type_name`), truthiness (`truthy`), `repr`/`str`, `json.dumps`/`json.loads` (`json::from_json` turns a `serde_json::Value` into a Python `Value`), `ast.literal_eval` and pickle
|
||||
- `litellm-core-utils` (`core_helpers`) for small utilities shared across crates, such as `json_type_name` and `unix_now`
|
||||
|
||||
If a helper you need is missing, add it to the owning crate instead of writing a private copy in a provider module
|
||||
|
||||
## Test placement
|
||||
|
||||
- Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;`
|
||||
|
|
|
|||
1
litellm-rust/Cargo.lock
generated
1
litellm-rust/Cargo.lock
generated
|
|
@ -4174,6 +4174,7 @@ dependencies = [
|
|||
"time",
|
||||
"tokio",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ repository.workspace = true
|
|||
test-support = ["litellm-http/test-support"]
|
||||
|
||||
[dependencies]
|
||||
uuid.workspace = true
|
||||
litellm-llms-types.workspace = true
|
||||
litellm-core-utils.workspace = true
|
||||
litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] }
|
||||
|
|
|
|||
|
|
@ -60,6 +60,10 @@ pub enum ErrorDetail {
|
|||
value: String,
|
||||
hint: String,
|
||||
},
|
||||
#[error(
|
||||
"Invalid type for 'tools[{index}].parameters': expected an object, but got {actual} instead."
|
||||
)]
|
||||
InvalidToolParameters { index: usize, actual: &'static str },
|
||||
#[error("invalid {subject} on line {line}: {source}")]
|
||||
InvalidLine {
|
||||
subject: &'static str,
|
||||
|
|
|
|||
|
|
@ -1 +1,2 @@
|
|||
pub mod request;
|
||||
pub mod transformation;
|
||||
|
|
|
|||
384
litellm-rust/crates/llms/src/openai/responses/request.rs
Normal file
384
litellm-rust/crates/llms/src/openai/responses/request.rs
Normal file
|
|
@ -0,0 +1,384 @@
|
|||
//! Request shaping shared by every host speaking the OpenAI Responses wire format, matching
|
||||
//! `litellm/llms/openai/responses/transformation.py`
|
||||
|
||||
use litellm_python_compat::{json::from_json, truthy::truthy};
|
||||
use serde_json::{Map, Value};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::{Error, ErrorDetail};
|
||||
|
||||
/// The Responses API rejects `max_output_tokens` below this for every model
|
||||
pub const MIN_MAX_OUTPUT_TOKENS: i64 = 16;
|
||||
|
||||
const REASONING_ITEM_ID_PREFIX: &str = "rs_";
|
||||
/// Python's `DEFAULT_MAX_RECURSE_DEPTH` for `filter_value_from_dict`
|
||||
const MAX_STRIP_DEPTH: usize = 100;
|
||||
const REASONING_FIELDS_DROPPED_WHEN_NULL: [&str; 3] = ["status", "content", "encrypted_content"];
|
||||
|
||||
/// Whether the host runs OpenAI's own item validator, which rejects reasoning it did not mint
|
||||
/// and tool-call item ids without its prefixes
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ItemValidation {
|
||||
OpenAi,
|
||||
Lenient,
|
||||
}
|
||||
|
||||
pub fn shape(
|
||||
input: Value,
|
||||
mut params: Map<String, Value>,
|
||||
validation: ItemValidation,
|
||||
) -> Result<(Value, Map<String, Value>), Error> {
|
||||
if let Some(max_output_tokens) = params.get_mut("max_output_tokens")
|
||||
&& max_output_tokens
|
||||
.as_i64()
|
||||
.is_some_and(|tokens| (1..MIN_MAX_OUTPUT_TOKENS).contains(&tokens))
|
||||
{
|
||||
*max_output_tokens = MIN_MAX_OUTPUT_TOKENS.into();
|
||||
}
|
||||
if let Some(Value::Array(tools)) = params.get_mut("tools") {
|
||||
tools
|
||||
.iter_mut()
|
||||
.filter_map(Value::as_object_mut)
|
||||
.for_each(|tool| strip_cache_control(tool, 0));
|
||||
tools
|
||||
.iter_mut()
|
||||
.enumerate()
|
||||
.try_for_each(|(index, tool)| decode_tool_parameters(index, tool))?;
|
||||
}
|
||||
let Value::Array(items) = input else {
|
||||
return Ok((input, params));
|
||||
};
|
||||
let items = items
|
||||
.into_iter()
|
||||
.filter(|item| validation == ItemValidation::Lenient || !is_minted_reasoning(item))
|
||||
.map(without_null_reasoning_fields)
|
||||
.map(|mut item| {
|
||||
if let Some(fields) = item.as_object_mut() {
|
||||
strip_cache_control(fields, 0);
|
||||
}
|
||||
item
|
||||
})
|
||||
.map(|item| match validation {
|
||||
ItemValidation::OpenAi => without_foreign_tool_call_id(item),
|
||||
ItemValidation::Lenient => item,
|
||||
})
|
||||
.collect();
|
||||
Ok((Value::Array(items), params))
|
||||
}
|
||||
|
||||
/// Anthropic-only `cache_control` marks, which OpenAI rejects as unknown parameters, removed at
|
||||
/// any depth like Python's `filter_value_from_dict`
|
||||
fn strip_cache_control(fields: &mut Map<String, Value>, depth: usize) {
|
||||
if depth > MAX_STRIP_DEPTH {
|
||||
return;
|
||||
}
|
||||
fields.remove("cache_control");
|
||||
for value in fields.values_mut() {
|
||||
match value {
|
||||
Value::Object(nested) => strip_cache_control(nested, depth + 1),
|
||||
Value::Array(items) => items
|
||||
.iter_mut()
|
||||
.filter_map(Value::as_object_mut)
|
||||
.for_each(|nested| strip_cache_control(nested, depth + 1)),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn decode_tool_parameters(index: usize, tool: &mut Value) -> Result<(), Error> {
|
||||
let Some(parameters) = tool
|
||||
.as_object_mut()
|
||||
.and_then(|tool| tool.get_mut("parameters"))
|
||||
else {
|
||||
return Ok(());
|
||||
};
|
||||
let decoded = match parameters {
|
||||
Value::Null | Value::Object(_) => return Ok(()),
|
||||
Value::String(encoded) => serde_json::from_str::<Value>(encoded)
|
||||
.ok()
|
||||
.filter(Value::is_object),
|
||||
_ => None,
|
||||
};
|
||||
let Some(decoded) = decoded else {
|
||||
return Err(Error::InvalidRequest(ErrorDetail::InvalidToolParameters {
|
||||
index,
|
||||
actual: from_json(parameters.clone()).type_name(),
|
||||
}));
|
||||
};
|
||||
*parameters = decoded;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reasoning LiteLLM minted while bridging another provider, which OpenAI cannot decrypt
|
||||
fn is_minted_reasoning(item: &Value) -> bool {
|
||||
let Some(item) = item.as_object() else {
|
||||
return false;
|
||||
};
|
||||
item.get("type").and_then(Value::as_str) == Some("reasoning")
|
||||
&& (item
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(is_minted_reasoning_item_id)
|
||||
|| item
|
||||
.get("encrypted_content")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(carries_thinking_blocks))
|
||||
}
|
||||
|
||||
fn is_minted_reasoning_item_id(id: &str) -> bool {
|
||||
id.strip_prefix(REASONING_ITEM_ID_PREFIX)
|
||||
.is_some_and(|suffix| {
|
||||
Uuid::try_parse(suffix).is_ok_and(|parsed| {
|
||||
parsed.get_version_num() == 4 && parsed.hyphenated().to_string() == suffix
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
fn carries_thinking_blocks(encrypted_content: &str) -> bool {
|
||||
let Ok(Value::Array(blocks)) = serde_json::from_str::<Value>(encrypted_content) else {
|
||||
return false;
|
||||
};
|
||||
blocks.iter().filter_map(Value::as_object).any(|block| {
|
||||
let is_set = |field: &str| {
|
||||
block
|
||||
.get(field)
|
||||
.is_some_and(|value| truthy(&from_json(value.clone())))
|
||||
};
|
||||
match block.get("type").and_then(Value::as_str) {
|
||||
Some("thinking") => is_set("signature"),
|
||||
Some("redacted_thinking") => is_set("data"),
|
||||
_ => false,
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/// OpenAI rejects `status: null` on a replayed reasoning item. Python round-trips a valid item
|
||||
/// through the SDK model with `exclude_none`, and otherwise drops only the nullable fields
|
||||
fn without_null_reasoning_fields(item: Value) -> Value {
|
||||
let Value::Object(mut fields) = item else {
|
||||
return item;
|
||||
};
|
||||
if fields.get("type").and_then(Value::as_str) != Some("reasoning") {
|
||||
return Value::Object(fields);
|
||||
}
|
||||
if is_sdk_reasoning_item(&fields) {
|
||||
drop_nulls(&mut fields);
|
||||
fields
|
||||
.iter_mut()
|
||||
.filter(|(name, _)| matches!(name.as_str(), "summary" | "content"))
|
||||
.filter_map(|(_, parts)| parts.as_array_mut())
|
||||
.flatten()
|
||||
.filter_map(Value::as_object_mut)
|
||||
.for_each(drop_nulls);
|
||||
} else {
|
||||
fields.retain(|name, value| {
|
||||
!(value.is_null() && REASONING_FIELDS_DROPPED_WHEN_NULL.contains(&name.as_str()))
|
||||
});
|
||||
}
|
||||
Value::Object(fields)
|
||||
}
|
||||
|
||||
/// Whether `ResponseReasoningItem.model_validate` accepts the item
|
||||
fn is_sdk_reasoning_item(fields: &Map<String, Value>) -> bool {
|
||||
let parts = |name: &str, part_type: &str, required: bool| match fields.get(name) {
|
||||
None | Some(Value::Null) => !required,
|
||||
Some(Value::Array(parts)) => parts.iter().all(|part| {
|
||||
part.get("type").and_then(Value::as_str) == Some(part_type)
|
||||
&& part.get("text").is_some_and(Value::is_string)
|
||||
}),
|
||||
Some(_) => false,
|
||||
};
|
||||
fields.get("id").is_some_and(Value::is_string)
|
||||
&& parts("summary", "summary_text", true)
|
||||
&& parts("content", "reasoning_text", false)
|
||||
&& fields
|
||||
.get("encrypted_content")
|
||||
.is_none_or(|value| value.is_null() || value.is_string())
|
||||
&& fields.get("status").is_none_or(|status| {
|
||||
status.is_null()
|
||||
|| matches!(
|
||||
status.as_str(),
|
||||
Some("in_progress" | "completed" | "incomplete")
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
fn drop_nulls(fields: &mut Map<String, Value>) {
|
||||
fields.retain(|_, value| !value.is_null());
|
||||
}
|
||||
|
||||
fn without_foreign_tool_call_id(item: Value) -> Value {
|
||||
let Value::Object(mut fields) = item else {
|
||||
return item;
|
||||
};
|
||||
let prefix = match fields.get("type").and_then(Value::as_str) {
|
||||
Some("function_call") => "fc",
|
||||
Some("custom_tool_call") => "ctc",
|
||||
_ => return Value::Object(fields),
|
||||
};
|
||||
if fields
|
||||
.get("id")
|
||||
.and_then(Value::as_str)
|
||||
.is_some_and(|id| !id.starts_with(prefix))
|
||||
{
|
||||
fields.remove("id");
|
||||
}
|
||||
Value::Object(fields)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rstest::rstest;
|
||||
use serde_json::json;
|
||||
|
||||
use super::*;
|
||||
|
||||
fn shaped(input: Value, params: Value, validation: ItemValidation) -> (Value, Value) {
|
||||
let Value::Object(params) = params else {
|
||||
panic!("params must be an object")
|
||||
};
|
||||
let (input, params) = shape(input, params, validation).unwrap();
|
||||
(input, Value::Object(params))
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::warm_up_probe(json!(1), json!(16))]
|
||||
#[case::just_below(json!(15), json!(16))]
|
||||
#[case::minimum(json!(16), json!(16))]
|
||||
#[case::zero_left_for_the_provider(json!(0), json!(0))]
|
||||
#[case::negative_left_for_the_provider(json!(-1), json!(-1))]
|
||||
#[case::null(Value::Null, Value::Null)]
|
||||
fn raises_max_output_tokens_to_the_minimum(#[case] requested: Value, #[case] sent: Value) {
|
||||
let (_, params) = shaped(
|
||||
json!("hi"),
|
||||
json!({"max_output_tokens": requested}),
|
||||
ItemValidation::Lenient,
|
||||
);
|
||||
assert_eq!(params, json!({"max_output_tokens": sent}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn strips_cache_control_at_any_depth() {
|
||||
let mark = json!({"type": "ephemeral"});
|
||||
let (input, params) = shaped(
|
||||
json!([
|
||||
{"role": "user", "content": "hi", "cache_control": mark},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "a", "cache_control": mark}]},
|
||||
{"type": "function_call_output", "call_id": "c", "output": [{"type": "input_text", "text": "b", "cache_control": mark}]}
|
||||
]),
|
||||
json!({"tools": [{"type": "function", "name": "a", "cache_control": mark, "parameters": {"type": "object"}}]}),
|
||||
ItemValidation::Lenient,
|
||||
);
|
||||
assert_eq!(
|
||||
(input, params),
|
||||
(
|
||||
json!([
|
||||
{"role": "user", "content": "hi"},
|
||||
{"role": "user", "content": [{"type": "input_text", "text": "a"}]},
|
||||
{"type": "function_call_output", "call_id": "c", "output": [{"type": "input_text", "text": "b"}]}
|
||||
]),
|
||||
json!({"tools": [{"type": "function", "name": "a", "parameters": {"type": "object"}}]})
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decodes_json_encoded_tool_parameters() {
|
||||
let (_, params) = shaped(
|
||||
json!("hi"),
|
||||
json!({"tools": [
|
||||
{"type": "function", "name": "a", "parameters": "{\"type\":\"object\"}"},
|
||||
{"type": "function", "name": "b", "parameters": {"type": "object"}},
|
||||
{"type": "function", "name": "c", "parameters": null},
|
||||
{"type": "web_search"}
|
||||
]}),
|
||||
ItemValidation::Lenient,
|
||||
);
|
||||
assert_eq!(
|
||||
params,
|
||||
json!({"tools": [
|
||||
{"type": "function", "name": "a", "parameters": {"type": "object"}},
|
||||
{"type": "function", "name": "b", "parameters": {"type": "object"}},
|
||||
{"type": "function", "name": "c", "parameters": null},
|
||||
{"type": "web_search"}
|
||||
]})
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::not_json(json!("{"), "str")]
|
||||
#[case::json_array(json!("[]"), "str")]
|
||||
#[case::number(json!(3), "int")]
|
||||
#[case::list(json!([]), "list")]
|
||||
fn rejects_tool_parameters_that_are_not_objects(
|
||||
#[case] parameters: Value,
|
||||
#[case] actual: &str,
|
||||
) {
|
||||
let Value::Object(params) = json!({"tools": [{"type": "web_search"}, {"type": "function", "parameters": parameters}]})
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
assert_eq!(
|
||||
shape(json!("hi"), params, ItemValidation::Lenient)
|
||||
.unwrap_err()
|
||||
.to_string(),
|
||||
format!(
|
||||
"invalid request: Invalid type for 'tools[1].parameters': expected an object, but got {actual} instead."
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::sdk_shaped(
|
||||
json!({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "x"}], "status": null, "encrypted_content": null, "extra": null}),
|
||||
json!({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "x"}]})
|
||||
)]
|
||||
#[case::missing_id_keeps_unknown_nulls(
|
||||
json!({"type": "reasoning", "summary": [], "status": null, "extra": null}),
|
||||
json!({"type": "reasoning", "summary": [], "extra": null})
|
||||
)]
|
||||
#[case::missing_summary(
|
||||
json!({"type": "reasoning", "id": "rs_1", "content": null}),
|
||||
json!({"type": "reasoning", "id": "rs_1"})
|
||||
)]
|
||||
#[case::not_reasoning(
|
||||
json!({"type": "message", "role": "user", "content": "hi", "status": null}),
|
||||
json!({"type": "message", "role": "user", "content": "hi", "status": null})
|
||||
)]
|
||||
fn drops_null_reasoning_fields(#[case] item: Value, #[case] expected: Value) {
|
||||
let (input, _) = shaped(json!([item]), json!({}), ItemValidation::Lenient);
|
||||
assert_eq!(input, json!([expected]));
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::minted_id(json!({"type": "reasoning", "id": "rs_6f9619ff-8b86-4d01-b42d-00c04fc964ff", "summary": []}), true)]
|
||||
#[case::bridged_thinking(
|
||||
json!({"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "[{\"type\":\"thinking\",\"thinking\":\"x\",\"signature\":\"sig\"}]"}),
|
||||
true
|
||||
)]
|
||||
#[case::unsigned_thinking(
|
||||
json!({"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "[{\"type\":\"thinking\",\"thinking\":\"x\"}]"}),
|
||||
false
|
||||
)]
|
||||
#[case::openai_id(json!({"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "encrypted_content": "gAAAA"}), false)]
|
||||
#[case::uppercase_uuid(json!({"type": "reasoning", "id": "rs_6F9619FF-8B86-4D01-B42D-00C04FC964FF", "summary": []}), false)]
|
||||
#[case::uuid_v1(json!({"type": "reasoning", "id": "rs_6f9619ff-8b86-11d0-b42d-00c04fc964ff", "summary": []}), false)]
|
||||
fn drops_minted_reasoning_for_openai(#[case] item: Value, #[case] minted: bool) {
|
||||
let input = json!([item, {"role": "user", "content": "hi"}]);
|
||||
let (openai, _) = shaped(input.clone(), json!({}), ItemValidation::OpenAi);
|
||||
let (lenient, _) = shaped(input.clone(), json!({}), ItemValidation::Lenient);
|
||||
assert_eq!(openai.as_array().unwrap().len(), if minted { 1 } else { 2 });
|
||||
assert_eq!(lenient, input);
|
||||
}
|
||||
|
||||
#[rstest]
|
||||
#[case::foreign_function_call(json!({"type": "function_call", "id": "toolu_1", "call_id": "c"}), json!({"type": "function_call", "call_id": "c"}))]
|
||||
#[case::openai_function_call(json!({"type": "function_call", "id": "fc_1", "call_id": "c"}), json!({"type": "function_call", "id": "fc_1", "call_id": "c"}))]
|
||||
#[case::foreign_custom_tool_call(json!({"type": "custom_tool_call", "id": "x", "call_id": "c"}), json!({"type": "custom_tool_call", "call_id": "c"}))]
|
||||
#[case::other_item(json!({"type": "message", "id": "x", "role": "user", "content": "hi"}), json!({"type": "message", "id": "x", "role": "user", "content": "hi"}))]
|
||||
fn drops_foreign_tool_call_ids_for_openai(#[case] item: Value, #[case] expected: Value) {
|
||||
let (openai, _) = shaped(json!([item.clone()]), json!({}), ItemValidation::OpenAi);
|
||||
let (lenient, _) = shaped(json!([item.clone()]), json!({}), ItemValidation::Lenient);
|
||||
assert_eq!((openai, lenient), (json!([expected]), json!([item])));
|
||||
}
|
||||
}
|
||||
|
|
@ -5,6 +5,7 @@ use serde_json::{Map, Value};
|
|||
|
||||
use litellm_auth::{CredentialPlacement, SecretValue};
|
||||
|
||||
use super::request::{self, ItemValidation};
|
||||
use crate::{
|
||||
Error,
|
||||
base_llm::{
|
||||
|
|
@ -165,7 +166,7 @@ impl BaseResponsesApiConfig for OpenAiResponsesApiConfig {
|
|||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
responses_request(model, input, params)
|
||||
responses_request(model, input, params, ItemValidation::OpenAi)
|
||||
}
|
||||
|
||||
fn transform_response_api_response(&self, body: Value) -> Result<ResponsesApiResponse, Error> {
|
||||
|
|
@ -178,6 +179,7 @@ pub fn responses_request(
|
|||
model: &str,
|
||||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
validation: ItemValidation,
|
||||
) -> Result<Value, Error> {
|
||||
if !input.is_string() && !input.is_array() {
|
||||
return Err(Error::InvalidRequest(
|
||||
|
|
@ -190,6 +192,7 @@ pub fn responses_request(
|
|||
{
|
||||
return Err(Error::InvalidRequest("stream must be a boolean".into()));
|
||||
}
|
||||
let (input, params) = request::shape(input, params, validation)?;
|
||||
Ok(Value::Object(
|
||||
params
|
||||
.into_iter()
|
||||
|
|
|
|||
|
|
@ -9,7 +9,10 @@ use crate::{
|
|||
auth::{AuthScheme, ValidatedEnvironment},
|
||||
responses::transformation::BaseResponsesApiConfig,
|
||||
},
|
||||
openai::responses::transformation::{responses_request, responses_response},
|
||||
openai::responses::{
|
||||
request::ItemValidation,
|
||||
transformation::{responses_request, responses_response},
|
||||
},
|
||||
};
|
||||
|
||||
const API_KEY_ENVS: [&str; 2] = ["OPENROUTER_API_KEY", "OR_API_KEY"];
|
||||
|
|
@ -77,7 +80,7 @@ impl BaseResponsesApiConfig for OpenRouterResponsesApiConfig {
|
|||
input: Value,
|
||||
params: Map<String, Value>,
|
||||
) -> Result<Value, Error> {
|
||||
responses_request(model, input, params)
|
||||
responses_request(model, input, params, ItemValidation::Lenient)
|
||||
}
|
||||
|
||||
fn transform_response_api_response(&self, body: Value) -> Result<ResponsesApiResponse, Error> {
|
||||
|
|
|
|||
|
|
@ -1,15 +1,18 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Mapping
|
||||
from typing import Final
|
||||
from typing import Final, cast # noqa: TID251 # narrows caller json the checker cannot see into
|
||||
|
||||
import litellm
|
||||
from litellm import get_llm_provider
|
||||
from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig
|
||||
from litellm.rust_bridge import failures
|
||||
from litellm.rust_bridge.public_call import inference_decline_reason, known_params, optional_str
|
||||
from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse
|
||||
from litellm.types.router import GenericLiteLLMParams
|
||||
|
||||
PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__)
|
||||
_OPENAI_CONFIG: Final = OpenAIResponsesAPIConfig()
|
||||
|
||||
|
||||
def body_params(request: Mapping[str, object]) -> Mapping[str, object]:
|
||||
|
|
@ -44,4 +47,52 @@ def decline_reason(request: Mapping[str, object]) -> str | None:
|
|||
return "native Responses could not resolve the provider"
|
||||
if provider != "openai":
|
||||
return "native HTTP responses provider"
|
||||
return inference_decline_reason(PARAMETERS, request)
|
||||
return inference_decline_reason(PARAMETERS, request) or _model_rules_decline_reason(request)
|
||||
|
||||
|
||||
def _model_rules_decline_reason(request: Mapping[str, object]) -> str | None:
|
||||
"""Request rewrites that read the model cost map or Python's regex engine stay on Python"""
|
||||
model: Final = str(request["model"]).removeprefix("openai/")
|
||||
params: Final = {name: request[name] for name in PARAMETERS if request.get(name) is not None}
|
||||
try:
|
||||
_OPENAI_CONFIG.map_openai_params(params, model, drop_params=False) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # the same rules the Python route applies
|
||||
except litellm.UnsupportedParamsError:
|
||||
return "native Responses does not implement the model's sampling rules"
|
||||
litellm_params: Final = GenericLiteLLMParams(
|
||||
api_base=optional_str(request.get("api_base")) or optional_str(request.get("base_url")),
|
||||
model_info=request.get("model_info"),
|
||||
)
|
||||
if _repairs_tool_schemas(model, request.get("tools"), litellm_params):
|
||||
return "native Responses does not implement tool schema repair"
|
||||
if _carries_cache_control(request.get("input")) and _OPENAI_CONFIG._reads_prompt_cache_breakpoints( # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies
|
||||
model=model, litellm_params=litellm_params, prompt_cache_options=request.get("prompt_cache_options")
|
||||
):
|
||||
return "native Responses does not implement prompt cache breakpoints"
|
||||
return None
|
||||
|
||||
|
||||
def _repairs_tool_schemas(model: str, tools: object, litellm_params: GenericLiteLLMParams) -> bool:
|
||||
if not isinstance(tools, list):
|
||||
return False
|
||||
try:
|
||||
decoded: Final = _OPENAI_CONFIG._tools_with_object_parameters(model=model, tools=cast(list[object], tools)) # pyright: ignore[reportPrivateUsage, reportArgumentType] # cast-ok: caller json
|
||||
except litellm.BadRequestError:
|
||||
return False
|
||||
return (
|
||||
_OPENAI_CONFIG._sanitized_tool_schemas_for_openai(model=model, tools=decoded, litellm_params=litellm_params) # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies
|
||||
is not decoded
|
||||
)
|
||||
|
||||
|
||||
def _carries_cache_control(value: object) -> bool:
|
||||
pending: Final[list[object]] = [value] # mutable-ok: work stack for a depth-first walk of caller json
|
||||
while pending:
|
||||
current = pending.pop()
|
||||
if isinstance(current, Mapping):
|
||||
fields = cast(Mapping[str, object], current) # cast-ok: caller json
|
||||
if "cache_control" in fields:
|
||||
return True
|
||||
pending.extend(fields.values())
|
||||
elif isinstance(current, list):
|
||||
pending.extend(cast(list[object], current)) # cast-ok: caller json
|
||||
return False
|
||||
|
|
|
|||
|
|
@ -156,6 +156,32 @@ AFFINITY_REPLAY_INPUT: Final[list[JsonValue]] = [
|
|||
{"role": "user", "content": "And Germany?"},
|
||||
]
|
||||
|
||||
MINTED_REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [
|
||||
{"type": "reasoning", "id": "rs_6f9619ff-8b86-4d01-b42d-00c04fc964ff", "summary": []},
|
||||
{
|
||||
"type": "reasoning",
|
||||
"id": "rs_0a1b2c3d4e5f",
|
||||
"summary": [],
|
||||
"encrypted_content": '[{"type":"thinking","thinking":"x","signature":"sig"}]',
|
||||
},
|
||||
{"role": "user", "content": "And Germany?"},
|
||||
]
|
||||
|
||||
FOREIGN_TOOL_CALL_REPLAY_INPUT: Final[list[JsonValue]] = [
|
||||
{
|
||||
"type": "function_call",
|
||||
"id": "toolu_6f5e4d3c2b1a",
|
||||
"call_id": "call_6f5e4d3c2b1a",
|
||||
"name": "get_weather",
|
||||
"arguments": '{"city":"Paris"}',
|
||||
},
|
||||
{"type": "function_call_output", "call_id": "call_6f5e4d3c2b1a", "output": "18C and sunny"},
|
||||
]
|
||||
|
||||
PATTERNED_PARAMETERS: Final[Mapping[str, JsonValue]] = MappingProxyType(
|
||||
{"type": "object", "properties": {"city": {"type": "string", "pattern": "^\\p{L}+$"}}}
|
||||
)
|
||||
|
||||
REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [
|
||||
{"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "status": None},
|
||||
{"role": "user", "content": "And Germany?"},
|
||||
|
|
@ -183,6 +209,7 @@ class Case:
|
|||
kwargs: Mapping[str, object]
|
||||
upstream: Mapping[str, JsonValue] = TEXT_RESPONSE
|
||||
gaps: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({}))
|
||||
falls_back: bool = False
|
||||
|
||||
def params(self) -> Iterator[object]:
|
||||
for aspect in ASPECTS:
|
||||
|
|
@ -196,10 +223,6 @@ DEPLOYMENT_METADATA: Final = MappingProxyType(
|
|||
)
|
||||
|
||||
|
||||
def _gaps(**gaps: str) -> Mapping[str, str]:
|
||||
return MappingProxyType(gaps)
|
||||
|
||||
|
||||
CASES: Final = (
|
||||
Case("plain_text", {"input": "What is the capital of France?"}),
|
||||
Case(
|
||||
|
|
@ -236,14 +259,11 @@ CASES: Final = (
|
|||
Case(
|
||||
"warm_up_probe_below_minimum_max_output_tokens",
|
||||
{"input": "hi", "max_output_tokens": 1},
|
||||
gaps=_gaps(request="native requests send max_output_tokens below OpenAI's minimum of 16"),
|
||||
),
|
||||
Case(
|
||||
"gpt_5_family_drops_temperature_while_reasoning",
|
||||
{"input": "hi", "temperature": 0.2, "reasoning": {"effort": "high"}, "drop_params": True},
|
||||
gaps=MappingProxyType(
|
||||
{"dispatch": "native responses decline drop_params instead of applying the GPT-5 sampling rules"}
|
||||
),
|
||||
falls_back=True,
|
||||
),
|
||||
Case(
|
||||
"anthropic_cache_control_is_stripped",
|
||||
|
|
@ -262,7 +282,6 @@ CASES: Final = (
|
|||
{
|
||||
"input": REASONING_REPLAY_INPUT,
|
||||
},
|
||||
gaps=_gaps(request="native requests forward reasoning items with status null"),
|
||||
),
|
||||
Case(
|
||||
"tool_parameters_sent_as_json_string",
|
||||
|
|
@ -270,7 +289,26 @@ CASES: Final = (
|
|||
"input": "Weather in Paris?",
|
||||
"tools": [{**WEATHER_TOOL, "parameters": '{"type":"object","properties":{"city":{"type":"string"}}}'}],
|
||||
},
|
||||
gaps=_gaps(request="native requests forward tool parameters encoded as a JSON string"),
|
||||
),
|
||||
Case(
|
||||
"item_level_cache_control_is_stripped",
|
||||
{"input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}]},
|
||||
),
|
||||
Case(
|
||||
"bridge_minted_reasoning_is_dropped",
|
||||
{"input": MINTED_REASONING_REPLAY_INPUT},
|
||||
),
|
||||
Case(
|
||||
"foreign_tool_call_ids_are_dropped",
|
||||
{"input": FOREIGN_TOOL_CALL_REPLAY_INPUT, "tools": [dict(WEATHER_TOOL)]},
|
||||
),
|
||||
Case(
|
||||
"tool_schema_repair_stays_on_python",
|
||||
{
|
||||
"input": "Weather in Paris?",
|
||||
"tools": [{**WEATHER_TOOL, "parameters": dict(PATTERNED_PARAMETERS)}],
|
||||
},
|
||||
falls_back=True,
|
||||
),
|
||||
)
|
||||
|
||||
|
|
@ -351,7 +389,7 @@ async def test_native_responses_match_python(
|
|||
native: Final = await _leg(case, parity_server, probe=native_responses)
|
||||
match aspect:
|
||||
case "dispatch":
|
||||
assert not native.fell_back
|
||||
assert native.fell_back == case.falls_back
|
||||
case "request":
|
||||
assert (native.request_body, native.request_headers) == (python.request_body, python.request_headers)
|
||||
case "response":
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ import pytest
|
|||
from pydantic import ValidationError
|
||||
|
||||
import litellm
|
||||
from litellm.rust_bridge.host.routes.responses import connection_defaults, map_failure, response
|
||||
from litellm.rust_bridge.host.routes.responses import connection_defaults, decline_reason, map_failure, response
|
||||
from litellm.types.llms.openai import ResponsesAPIResponse
|
||||
|
||||
|
||||
|
|
@ -87,3 +87,49 @@ def test_failure_preserves_the_explicit_endpoint(api_base: str | None, base_url:
|
|||
assert isinstance(mapped, litellm.RateLimitError)
|
||||
assert str(mapped.response.request.url) == expected
|
||||
assert mapped.__context__ is upstream
|
||||
|
||||
|
||||
_PATTERNED_TOOL: Final = MappingProxyType(
|
||||
{
|
||||
"type": "function",
|
||||
"name": "lookup",
|
||||
"parameters": {"type": "object", "properties": {"city": {"type": "string", "pattern": "^\\p{L}+$"}}},
|
||||
}
|
||||
)
|
||||
_PLAIN_TOOL: Final = MappingProxyType(
|
||||
{"type": "function", "name": "lookup", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}}
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_fields", "expected"),
|
||||
(
|
||||
pytest.param({"input": "hi", "temperature": 1}, None, id="supported_temperature"),
|
||||
pytest.param(
|
||||
{"input": "hi", "temperature": 0.2},
|
||||
"native Responses does not implement the model's sampling rules",
|
||||
id="gpt_5_sampling_rules",
|
||||
),
|
||||
pytest.param({"input": "hi", "tools": [dict(_PLAIN_TOOL)]}, None, id="valid_tool_schema"),
|
||||
pytest.param(
|
||||
{"input": "hi", "tools": [dict(_PATTERNED_TOOL)]},
|
||||
"native Responses does not implement tool schema repair",
|
||||
id="python_incompatible_pattern",
|
||||
),
|
||||
pytest.param(
|
||||
{"input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}]},
|
||||
"native Responses does not implement prompt cache breakpoints",
|
||||
id="prompt_cache_breakpoints_on_openai",
|
||||
),
|
||||
pytest.param(
|
||||
{
|
||||
"input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}],
|
||||
"api_base": "http://127.0.0.1:9/v1",
|
||||
},
|
||||
None,
|
||||
id="cache_control_on_a_compatible_host",
|
||||
),
|
||||
),
|
||||
)
|
||||
def test_model_rules_stay_on_python(request_fields: dict[str, object], expected: str | None) -> None:
|
||||
assert decline_reason(MappingProxyType({"model": "openai/gpt-6.1-sol", **request_fields})) == expected
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue