diff --git a/litellm-rust/AGENTS.md b/litellm-rust/AGENTS.md index e62a9463cb5..58f8d4dd0ba 100644 --- a/litellm-rust/AGENTS.md +++ b/litellm-rust/AGENTS.md @@ -8,6 +8,15 @@ For fieldless enums, derive `strum::VariantArray` and use `VARIANTS` instead of Use the derived conversions directly (`<&'static str>::from(x)` / `.into()`, `str::parse`) with no `as_str`/`parse` wrapper that only delegates, and spell each variant with explicit `#[strum(serialize = "...")]` instead of `serialize_all` +## Existing helpers + +Before writing a small helper that reproduces Python behaviour or inspects JSON, look for it where it belongs and reuse it: + +- `litellm-python-compat` for Python semantics over plain data: type names (`Value::type_name`), truthiness (`truthy`), `repr`/`str`, `json.dumps`/`json.loads` (`json::from_json` turns a `serde_json::Value` into a Python `Value`), `ast.literal_eval` and pickle +- `litellm-core-utils` (`core_helpers`) for small utilities shared across crates, such as `json_type_name` and `unix_now` + +If a helper you need is missing, add it to the owning crate instead of writing a private copy in a provider module + ## Test placement - Never create a `tests.rs` (or `test.rs`) file under `src/`, and never `#[path = "tests.rs"] mod tests;` diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 98f36607fff..a736315ed08 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -4174,6 +4174,7 @@ dependencies = [ "time", "tokio", "url", + "uuid", ] [[package]] diff --git a/litellm-rust/crates/llms/Cargo.toml b/litellm-rust/crates/llms/Cargo.toml index 2d3ea85d1fc..f595f712023 100644 --- a/litellm-rust/crates/llms/Cargo.toml +++ b/litellm-rust/crates/llms/Cargo.toml @@ -9,6 +9,7 @@ repository.workspace = true test-support = ["litellm-http/test-support"] [dependencies] +uuid.workspace = true litellm-llms-types.workspace = true litellm-core-utils.workspace = true litellm-auth = { workspace = true, features = ["aws", "azure", "gcp"] } diff --git a/litellm-rust/crates/llms/src/error.rs b/litellm-rust/crates/llms/src/error.rs index 10a4f696406..799d0cd03df 100644 --- a/litellm-rust/crates/llms/src/error.rs +++ b/litellm-rust/crates/llms/src/error.rs @@ -60,6 +60,10 @@ pub enum ErrorDetail { value: String, hint: String, }, + #[error( + "Invalid type for 'tools[{index}].parameters': expected an object, but got {actual} instead." + )] + InvalidToolParameters { index: usize, actual: &'static str }, #[error("invalid {subject} on line {line}: {source}")] InvalidLine { subject: &'static str, diff --git a/litellm-rust/crates/llms/src/openai/responses/mod.rs b/litellm-rust/crates/llms/src/openai/responses/mod.rs index f239b6921fa..bac848cee6b 100644 --- a/litellm-rust/crates/llms/src/openai/responses/mod.rs +++ b/litellm-rust/crates/llms/src/openai/responses/mod.rs @@ -1 +1,2 @@ +pub mod request; pub mod transformation; diff --git a/litellm-rust/crates/llms/src/openai/responses/request.rs b/litellm-rust/crates/llms/src/openai/responses/request.rs new file mode 100644 index 00000000000..b8fe592c0e2 --- /dev/null +++ b/litellm-rust/crates/llms/src/openai/responses/request.rs @@ -0,0 +1,384 @@ +//! Request shaping shared by every host speaking the OpenAI Responses wire format, matching +//! `litellm/llms/openai/responses/transformation.py` + +use litellm_python_compat::{json::from_json, truthy::truthy}; +use serde_json::{Map, Value}; +use uuid::Uuid; + +use crate::{Error, ErrorDetail}; + +/// The Responses API rejects `max_output_tokens` below this for every model +pub const MIN_MAX_OUTPUT_TOKENS: i64 = 16; + +const REASONING_ITEM_ID_PREFIX: &str = "rs_"; +/// Python's `DEFAULT_MAX_RECURSE_DEPTH` for `filter_value_from_dict` +const MAX_STRIP_DEPTH: usize = 100; +const REASONING_FIELDS_DROPPED_WHEN_NULL: [&str; 3] = ["status", "content", "encrypted_content"]; + +/// Whether the host runs OpenAI's own item validator, which rejects reasoning it did not mint +/// and tool-call item ids without its prefixes +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ItemValidation { + OpenAi, + Lenient, +} + +pub fn shape( + input: Value, + mut params: Map, + validation: ItemValidation, +) -> Result<(Value, Map), Error> { + if let Some(max_output_tokens) = params.get_mut("max_output_tokens") + && max_output_tokens + .as_i64() + .is_some_and(|tokens| (1..MIN_MAX_OUTPUT_TOKENS).contains(&tokens)) + { + *max_output_tokens = MIN_MAX_OUTPUT_TOKENS.into(); + } + if let Some(Value::Array(tools)) = params.get_mut("tools") { + tools + .iter_mut() + .filter_map(Value::as_object_mut) + .for_each(|tool| strip_cache_control(tool, 0)); + tools + .iter_mut() + .enumerate() + .try_for_each(|(index, tool)| decode_tool_parameters(index, tool))?; + } + let Value::Array(items) = input else { + return Ok((input, params)); + }; + let items = items + .into_iter() + .filter(|item| validation == ItemValidation::Lenient || !is_minted_reasoning(item)) + .map(without_null_reasoning_fields) + .map(|mut item| { + if let Some(fields) = item.as_object_mut() { + strip_cache_control(fields, 0); + } + item + }) + .map(|item| match validation { + ItemValidation::OpenAi => without_foreign_tool_call_id(item), + ItemValidation::Lenient => item, + }) + .collect(); + Ok((Value::Array(items), params)) +} + +/// Anthropic-only `cache_control` marks, which OpenAI rejects as unknown parameters, removed at +/// any depth like Python's `filter_value_from_dict` +fn strip_cache_control(fields: &mut Map, depth: usize) { + if depth > MAX_STRIP_DEPTH { + return; + } + fields.remove("cache_control"); + for value in fields.values_mut() { + match value { + Value::Object(nested) => strip_cache_control(nested, depth + 1), + Value::Array(items) => items + .iter_mut() + .filter_map(Value::as_object_mut) + .for_each(|nested| strip_cache_control(nested, depth + 1)), + _ => {} + } + } +} + +fn decode_tool_parameters(index: usize, tool: &mut Value) -> Result<(), Error> { + let Some(parameters) = tool + .as_object_mut() + .and_then(|tool| tool.get_mut("parameters")) + else { + return Ok(()); + }; + let decoded = match parameters { + Value::Null | Value::Object(_) => return Ok(()), + Value::String(encoded) => serde_json::from_str::(encoded) + .ok() + .filter(Value::is_object), + _ => None, + }; + let Some(decoded) = decoded else { + return Err(Error::InvalidRequest(ErrorDetail::InvalidToolParameters { + index, + actual: from_json(parameters.clone()).type_name(), + })); + }; + *parameters = decoded; + Ok(()) +} + +/// Reasoning LiteLLM minted while bridging another provider, which OpenAI cannot decrypt +fn is_minted_reasoning(item: &Value) -> bool { + let Some(item) = item.as_object() else { + return false; + }; + item.get("type").and_then(Value::as_str) == Some("reasoning") + && (item + .get("id") + .and_then(Value::as_str) + .is_some_and(is_minted_reasoning_item_id) + || item + .get("encrypted_content") + .and_then(Value::as_str) + .is_some_and(carries_thinking_blocks)) +} + +fn is_minted_reasoning_item_id(id: &str) -> bool { + id.strip_prefix(REASONING_ITEM_ID_PREFIX) + .is_some_and(|suffix| { + Uuid::try_parse(suffix).is_ok_and(|parsed| { + parsed.get_version_num() == 4 && parsed.hyphenated().to_string() == suffix + }) + }) +} + +fn carries_thinking_blocks(encrypted_content: &str) -> bool { + let Ok(Value::Array(blocks)) = serde_json::from_str::(encrypted_content) else { + return false; + }; + blocks.iter().filter_map(Value::as_object).any(|block| { + let is_set = |field: &str| { + block + .get(field) + .is_some_and(|value| truthy(&from_json(value.clone()))) + }; + match block.get("type").and_then(Value::as_str) { + Some("thinking") => is_set("signature"), + Some("redacted_thinking") => is_set("data"), + _ => false, + } + }) +} + +/// OpenAI rejects `status: null` on a replayed reasoning item. Python round-trips a valid item +/// through the SDK model with `exclude_none`, and otherwise drops only the nullable fields +fn without_null_reasoning_fields(item: Value) -> Value { + let Value::Object(mut fields) = item else { + return item; + }; + if fields.get("type").and_then(Value::as_str) != Some("reasoning") { + return Value::Object(fields); + } + if is_sdk_reasoning_item(&fields) { + drop_nulls(&mut fields); + fields + .iter_mut() + .filter(|(name, _)| matches!(name.as_str(), "summary" | "content")) + .filter_map(|(_, parts)| parts.as_array_mut()) + .flatten() + .filter_map(Value::as_object_mut) + .for_each(drop_nulls); + } else { + fields.retain(|name, value| { + !(value.is_null() && REASONING_FIELDS_DROPPED_WHEN_NULL.contains(&name.as_str())) + }); + } + Value::Object(fields) +} + +/// Whether `ResponseReasoningItem.model_validate` accepts the item +fn is_sdk_reasoning_item(fields: &Map) -> bool { + let parts = |name: &str, part_type: &str, required: bool| match fields.get(name) { + None | Some(Value::Null) => !required, + Some(Value::Array(parts)) => parts.iter().all(|part| { + part.get("type").and_then(Value::as_str) == Some(part_type) + && part.get("text").is_some_and(Value::is_string) + }), + Some(_) => false, + }; + fields.get("id").is_some_and(Value::is_string) + && parts("summary", "summary_text", true) + && parts("content", "reasoning_text", false) + && fields + .get("encrypted_content") + .is_none_or(|value| value.is_null() || value.is_string()) + && fields.get("status").is_none_or(|status| { + status.is_null() + || matches!( + status.as_str(), + Some("in_progress" | "completed" | "incomplete") + ) + }) +} + +fn drop_nulls(fields: &mut Map) { + fields.retain(|_, value| !value.is_null()); +} + +fn without_foreign_tool_call_id(item: Value) -> Value { + let Value::Object(mut fields) = item else { + return item; + }; + let prefix = match fields.get("type").and_then(Value::as_str) { + Some("function_call") => "fc", + Some("custom_tool_call") => "ctc", + _ => return Value::Object(fields), + }; + if fields + .get("id") + .and_then(Value::as_str) + .is_some_and(|id| !id.starts_with(prefix)) + { + fields.remove("id"); + } + Value::Object(fields) +} + +#[cfg(test)] +mod tests { + use rstest::rstest; + use serde_json::json; + + use super::*; + + fn shaped(input: Value, params: Value, validation: ItemValidation) -> (Value, Value) { + let Value::Object(params) = params else { + panic!("params must be an object") + }; + let (input, params) = shape(input, params, validation).unwrap(); + (input, Value::Object(params)) + } + + #[rstest] + #[case::warm_up_probe(json!(1), json!(16))] + #[case::just_below(json!(15), json!(16))] + #[case::minimum(json!(16), json!(16))] + #[case::zero_left_for_the_provider(json!(0), json!(0))] + #[case::negative_left_for_the_provider(json!(-1), json!(-1))] + #[case::null(Value::Null, Value::Null)] + fn raises_max_output_tokens_to_the_minimum(#[case] requested: Value, #[case] sent: Value) { + let (_, params) = shaped( + json!("hi"), + json!({"max_output_tokens": requested}), + ItemValidation::Lenient, + ); + assert_eq!(params, json!({"max_output_tokens": sent})); + } + + #[test] + fn strips_cache_control_at_any_depth() { + let mark = json!({"type": "ephemeral"}); + let (input, params) = shaped( + json!([ + {"role": "user", "content": "hi", "cache_control": mark}, + {"role": "user", "content": [{"type": "input_text", "text": "a", "cache_control": mark}]}, + {"type": "function_call_output", "call_id": "c", "output": [{"type": "input_text", "text": "b", "cache_control": mark}]} + ]), + json!({"tools": [{"type": "function", "name": "a", "cache_control": mark, "parameters": {"type": "object"}}]}), + ItemValidation::Lenient, + ); + assert_eq!( + (input, params), + ( + json!([ + {"role": "user", "content": "hi"}, + {"role": "user", "content": [{"type": "input_text", "text": "a"}]}, + {"type": "function_call_output", "call_id": "c", "output": [{"type": "input_text", "text": "b"}]} + ]), + json!({"tools": [{"type": "function", "name": "a", "parameters": {"type": "object"}}]}) + ) + ); + } + + #[test] + fn decodes_json_encoded_tool_parameters() { + let (_, params) = shaped( + json!("hi"), + json!({"tools": [ + {"type": "function", "name": "a", "parameters": "{\"type\":\"object\"}"}, + {"type": "function", "name": "b", "parameters": {"type": "object"}}, + {"type": "function", "name": "c", "parameters": null}, + {"type": "web_search"} + ]}), + ItemValidation::Lenient, + ); + assert_eq!( + params, + json!({"tools": [ + {"type": "function", "name": "a", "parameters": {"type": "object"}}, + {"type": "function", "name": "b", "parameters": {"type": "object"}}, + {"type": "function", "name": "c", "parameters": null}, + {"type": "web_search"} + ]}) + ); + } + + #[rstest] + #[case::not_json(json!("{"), "str")] + #[case::json_array(json!("[]"), "str")] + #[case::number(json!(3), "int")] + #[case::list(json!([]), "list")] + fn rejects_tool_parameters_that_are_not_objects( + #[case] parameters: Value, + #[case] actual: &str, + ) { + let Value::Object(params) = json!({"tools": [{"type": "web_search"}, {"type": "function", "parameters": parameters}]}) + else { + unreachable!() + }; + assert_eq!( + shape(json!("hi"), params, ItemValidation::Lenient) + .unwrap_err() + .to_string(), + format!( + "invalid request: Invalid type for 'tools[1].parameters': expected an object, but got {actual} instead." + ) + ); + } + + #[rstest] + #[case::sdk_shaped( + json!({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "x"}], "status": null, "encrypted_content": null, "extra": null}), + json!({"type": "reasoning", "id": "rs_1", "summary": [{"type": "summary_text", "text": "x"}]}) + )] + #[case::missing_id_keeps_unknown_nulls( + json!({"type": "reasoning", "summary": [], "status": null, "extra": null}), + json!({"type": "reasoning", "summary": [], "extra": null}) + )] + #[case::missing_summary( + json!({"type": "reasoning", "id": "rs_1", "content": null}), + json!({"type": "reasoning", "id": "rs_1"}) + )] + #[case::not_reasoning( + json!({"type": "message", "role": "user", "content": "hi", "status": null}), + json!({"type": "message", "role": "user", "content": "hi", "status": null}) + )] + fn drops_null_reasoning_fields(#[case] item: Value, #[case] expected: Value) { + let (input, _) = shaped(json!([item]), json!({}), ItemValidation::Lenient); + assert_eq!(input, json!([expected])); + } + + #[rstest] + #[case::minted_id(json!({"type": "reasoning", "id": "rs_6f9619ff-8b86-4d01-b42d-00c04fc964ff", "summary": []}), true)] + #[case::bridged_thinking( + json!({"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "[{\"type\":\"thinking\",\"thinking\":\"x\",\"signature\":\"sig\"}]"}), + true + )] + #[case::unsigned_thinking( + json!({"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "[{\"type\":\"thinking\",\"thinking\":\"x\"}]"}), + false + )] + #[case::openai_id(json!({"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "encrypted_content": "gAAAA"}), false)] + #[case::uppercase_uuid(json!({"type": "reasoning", "id": "rs_6F9619FF-8B86-4D01-B42D-00C04FC964FF", "summary": []}), false)] + #[case::uuid_v1(json!({"type": "reasoning", "id": "rs_6f9619ff-8b86-11d0-b42d-00c04fc964ff", "summary": []}), false)] + fn drops_minted_reasoning_for_openai(#[case] item: Value, #[case] minted: bool) { + let input = json!([item, {"role": "user", "content": "hi"}]); + let (openai, _) = shaped(input.clone(), json!({}), ItemValidation::OpenAi); + let (lenient, _) = shaped(input.clone(), json!({}), ItemValidation::Lenient); + assert_eq!(openai.as_array().unwrap().len(), if minted { 1 } else { 2 }); + assert_eq!(lenient, input); + } + + #[rstest] + #[case::foreign_function_call(json!({"type": "function_call", "id": "toolu_1", "call_id": "c"}), json!({"type": "function_call", "call_id": "c"}))] + #[case::openai_function_call(json!({"type": "function_call", "id": "fc_1", "call_id": "c"}), json!({"type": "function_call", "id": "fc_1", "call_id": "c"}))] + #[case::foreign_custom_tool_call(json!({"type": "custom_tool_call", "id": "x", "call_id": "c"}), json!({"type": "custom_tool_call", "call_id": "c"}))] + #[case::other_item(json!({"type": "message", "id": "x", "role": "user", "content": "hi"}), json!({"type": "message", "id": "x", "role": "user", "content": "hi"}))] + fn drops_foreign_tool_call_ids_for_openai(#[case] item: Value, #[case] expected: Value) { + let (openai, _) = shaped(json!([item.clone()]), json!({}), ItemValidation::OpenAi); + let (lenient, _) = shaped(json!([item.clone()]), json!({}), ItemValidation::Lenient); + assert_eq!((openai, lenient), (json!([expected]), json!([item]))); + } +} diff --git a/litellm-rust/crates/llms/src/openai/responses/transformation.rs b/litellm-rust/crates/llms/src/openai/responses/transformation.rs index 99cc3559514..42c12b6ac58 100644 --- a/litellm-rust/crates/llms/src/openai/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openai/responses/transformation.rs @@ -5,6 +5,7 @@ use serde_json::{Map, Value}; use litellm_auth::{CredentialPlacement, SecretValue}; +use super::request::{self, ItemValidation}; use crate::{ Error, base_llm::{ @@ -165,7 +166,7 @@ impl BaseResponsesApiConfig for OpenAiResponsesApiConfig { input: Value, params: Map, ) -> Result { - responses_request(model, input, params) + responses_request(model, input, params, ItemValidation::OpenAi) } fn transform_response_api_response(&self, body: Value) -> Result { @@ -178,6 +179,7 @@ pub fn responses_request( model: &str, input: Value, params: Map, + validation: ItemValidation, ) -> Result { if !input.is_string() && !input.is_array() { return Err(Error::InvalidRequest( @@ -190,6 +192,7 @@ pub fn responses_request( { return Err(Error::InvalidRequest("stream must be a boolean".into())); } + let (input, params) = request::shape(input, params, validation)?; Ok(Value::Object( params .into_iter() diff --git a/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs b/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs index 900fba9e916..b51158571d0 100644 --- a/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs +++ b/litellm-rust/crates/llms/src/openrouter/responses/transformation.rs @@ -9,7 +9,10 @@ use crate::{ auth::{AuthScheme, ValidatedEnvironment}, responses::transformation::BaseResponsesApiConfig, }, - openai::responses::transformation::{responses_request, responses_response}, + openai::responses::{ + request::ItemValidation, + transformation::{responses_request, responses_response}, + }, }; const API_KEY_ENVS: [&str; 2] = ["OPENROUTER_API_KEY", "OR_API_KEY"]; @@ -77,7 +80,7 @@ impl BaseResponsesApiConfig for OpenRouterResponsesApiConfig { input: Value, params: Map, ) -> Result { - responses_request(model, input, params) + responses_request(model, input, params, ItemValidation::Lenient) } fn transform_response_api_response(&self, body: Value) -> Result { diff --git a/litellm/rust_bridge/host/routes/responses.py b/litellm/rust_bridge/host/routes/responses.py index 470a89a0e05..fea156fb8ff 100644 --- a/litellm/rust_bridge/host/routes/responses.py +++ b/litellm/rust_bridge/host/routes/responses.py @@ -1,15 +1,18 @@ from __future__ import annotations from collections.abc import Mapping -from typing import Final +from typing import Final, cast # noqa: TID251 # narrows caller json the checker cannot see into import litellm from litellm import get_llm_provider +from litellm.llms.openai.responses.transformation import OpenAIResponsesAPIConfig from litellm.rust_bridge import failures from litellm.rust_bridge.public_call import inference_decline_reason, known_params, optional_str from litellm.types.llms.openai import ResponsesAPIOptionalRequestParams, ResponsesAPIResponse +from litellm.types.router import GenericLiteLLMParams PARAMETERS: Final = tuple(ResponsesAPIOptionalRequestParams.__annotations__) +_OPENAI_CONFIG: Final = OpenAIResponsesAPIConfig() def body_params(request: Mapping[str, object]) -> Mapping[str, object]: @@ -44,4 +47,52 @@ def decline_reason(request: Mapping[str, object]) -> str | None: return "native Responses could not resolve the provider" if provider != "openai": return "native HTTP responses provider" - return inference_decline_reason(PARAMETERS, request) + return inference_decline_reason(PARAMETERS, request) or _model_rules_decline_reason(request) + + +def _model_rules_decline_reason(request: Mapping[str, object]) -> str | None: + """Request rewrites that read the model cost map or Python's regex engine stay on Python""" + model: Final = str(request["model"]).removeprefix("openai/") + params: Final = {name: request[name] for name in PARAMETERS if request.get(name) is not None} + try: + _OPENAI_CONFIG.map_openai_params(params, model, drop_params=False) # pyright: ignore[reportArgumentType, reportUnknownMemberType] # the same rules the Python route applies + except litellm.UnsupportedParamsError: + return "native Responses does not implement the model's sampling rules" + litellm_params: Final = GenericLiteLLMParams( + api_base=optional_str(request.get("api_base")) or optional_str(request.get("base_url")), + model_info=request.get("model_info"), + ) + if _repairs_tool_schemas(model, request.get("tools"), litellm_params): + return "native Responses does not implement tool schema repair" + if _carries_cache_control(request.get("input")) and _OPENAI_CONFIG._reads_prompt_cache_breakpoints( # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies + model=model, litellm_params=litellm_params, prompt_cache_options=request.get("prompt_cache_options") + ): + return "native Responses does not implement prompt cache breakpoints" + return None + + +def _repairs_tool_schemas(model: str, tools: object, litellm_params: GenericLiteLLMParams) -> bool: + if not isinstance(tools, list): + return False + try: + decoded: Final = _OPENAI_CONFIG._tools_with_object_parameters(model=model, tools=cast(list[object], tools)) # pyright: ignore[reportPrivateUsage, reportArgumentType] # cast-ok: caller json + except litellm.BadRequestError: + return False + return ( + _OPENAI_CONFIG._sanitized_tool_schemas_for_openai(model=model, tools=decoded, litellm_params=litellm_params) # pyright: ignore[reportPrivateUsage] # the same rules the Python route applies + is not decoded + ) + + +def _carries_cache_control(value: object) -> bool: + pending: Final[list[object]] = [value] # mutable-ok: work stack for a depth-first walk of caller json + while pending: + current = pending.pop() + if isinstance(current, Mapping): + fields = cast(Mapping[str, object], current) # cast-ok: caller json + if "cache_control" in fields: + return True + pending.extend(fields.values()) + elif isinstance(current, list): + pending.extend(cast(list[object], current)) # cast-ok: caller json + return False diff --git a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py index 0ed5cf0d01d..684eca0a482 100644 --- a/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py +++ b/tests/rust-python-harness/strategies/e2e_parity/sdk/responses/test_native_parity.py @@ -156,6 +156,32 @@ AFFINITY_REPLAY_INPUT: Final[list[JsonValue]] = [ {"role": "user", "content": "And Germany?"}, ] +MINTED_REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [ + {"type": "reasoning", "id": "rs_6f9619ff-8b86-4d01-b42d-00c04fc964ff", "summary": []}, + { + "type": "reasoning", + "id": "rs_0a1b2c3d4e5f", + "summary": [], + "encrypted_content": '[{"type":"thinking","thinking":"x","signature":"sig"}]', + }, + {"role": "user", "content": "And Germany?"}, +] + +FOREIGN_TOOL_CALL_REPLAY_INPUT: Final[list[JsonValue]] = [ + { + "type": "function_call", + "id": "toolu_6f5e4d3c2b1a", + "call_id": "call_6f5e4d3c2b1a", + "name": "get_weather", + "arguments": '{"city":"Paris"}', + }, + {"type": "function_call_output", "call_id": "call_6f5e4d3c2b1a", "output": "18C and sunny"}, +] + +PATTERNED_PARAMETERS: Final[Mapping[str, JsonValue]] = MappingProxyType( + {"type": "object", "properties": {"city": {"type": "string", "pattern": "^\\p{L}+$"}}} +) + REASONING_REPLAY_INPUT: Final[list[JsonValue]] = [ {"type": "reasoning", "id": "rs_0a1b2c3d4e5f", "summary": [], "status": None}, {"role": "user", "content": "And Germany?"}, @@ -183,6 +209,7 @@ class Case: kwargs: Mapping[str, object] upstream: Mapping[str, JsonValue] = TEXT_RESPONSE gaps: Mapping[str, str] = field(default_factory=lambda: MappingProxyType({})) + falls_back: bool = False def params(self) -> Iterator[object]: for aspect in ASPECTS: @@ -196,10 +223,6 @@ DEPLOYMENT_METADATA: Final = MappingProxyType( ) -def _gaps(**gaps: str) -> Mapping[str, str]: - return MappingProxyType(gaps) - - CASES: Final = ( Case("plain_text", {"input": "What is the capital of France?"}), Case( @@ -236,14 +259,11 @@ CASES: Final = ( Case( "warm_up_probe_below_minimum_max_output_tokens", {"input": "hi", "max_output_tokens": 1}, - gaps=_gaps(request="native requests send max_output_tokens below OpenAI's minimum of 16"), ), Case( "gpt_5_family_drops_temperature_while_reasoning", {"input": "hi", "temperature": 0.2, "reasoning": {"effort": "high"}, "drop_params": True}, - gaps=MappingProxyType( - {"dispatch": "native responses decline drop_params instead of applying the GPT-5 sampling rules"} - ), + falls_back=True, ), Case( "anthropic_cache_control_is_stripped", @@ -262,7 +282,6 @@ CASES: Final = ( { "input": REASONING_REPLAY_INPUT, }, - gaps=_gaps(request="native requests forward reasoning items with status null"), ), Case( "tool_parameters_sent_as_json_string", @@ -270,7 +289,26 @@ CASES: Final = ( "input": "Weather in Paris?", "tools": [{**WEATHER_TOOL, "parameters": '{"type":"object","properties":{"city":{"type":"string"}}}'}], }, - gaps=_gaps(request="native requests forward tool parameters encoded as a JSON string"), + ), + Case( + "item_level_cache_control_is_stripped", + {"input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}]}, + ), + Case( + "bridge_minted_reasoning_is_dropped", + {"input": MINTED_REASONING_REPLAY_INPUT}, + ), + Case( + "foreign_tool_call_ids_are_dropped", + {"input": FOREIGN_TOOL_CALL_REPLAY_INPUT, "tools": [dict(WEATHER_TOOL)]}, + ), + Case( + "tool_schema_repair_stays_on_python", + { + "input": "Weather in Paris?", + "tools": [{**WEATHER_TOOL, "parameters": dict(PATTERNED_PARAMETERS)}], + }, + falls_back=True, ), ) @@ -351,7 +389,7 @@ async def test_native_responses_match_python( native: Final = await _leg(case, parity_server, probe=native_responses) match aspect: case "dispatch": - assert not native.fell_back + assert native.fell_back == case.falls_back case "request": assert (native.request_body, native.request_headers) == (python.request_body, python.request_headers) case "response": diff --git a/tests/unit/rust_bridge/host/routes/test_responses.py b/tests/unit/rust_bridge/host/routes/test_responses.py index 99e3726c03a..f11c3d0ed21 100644 --- a/tests/unit/rust_bridge/host/routes/test_responses.py +++ b/tests/unit/rust_bridge/host/routes/test_responses.py @@ -5,7 +5,7 @@ import pytest from pydantic import ValidationError import litellm -from litellm.rust_bridge.host.routes.responses import connection_defaults, map_failure, response +from litellm.rust_bridge.host.routes.responses import connection_defaults, decline_reason, map_failure, response from litellm.types.llms.openai import ResponsesAPIResponse @@ -87,3 +87,49 @@ def test_failure_preserves_the_explicit_endpoint(api_base: str | None, base_url: assert isinstance(mapped, litellm.RateLimitError) assert str(mapped.response.request.url) == expected assert mapped.__context__ is upstream + + +_PATTERNED_TOOL: Final = MappingProxyType( + { + "type": "function", + "name": "lookup", + "parameters": {"type": "object", "properties": {"city": {"type": "string", "pattern": "^\\p{L}+$"}}}, + } +) +_PLAIN_TOOL: Final = MappingProxyType( + {"type": "function", "name": "lookup", "parameters": {"type": "object", "properties": {"city": {"type": "string"}}}} +) + + +@pytest.mark.parametrize( + ("request_fields", "expected"), + ( + pytest.param({"input": "hi", "temperature": 1}, None, id="supported_temperature"), + pytest.param( + {"input": "hi", "temperature": 0.2}, + "native Responses does not implement the model's sampling rules", + id="gpt_5_sampling_rules", + ), + pytest.param({"input": "hi", "tools": [dict(_PLAIN_TOOL)]}, None, id="valid_tool_schema"), + pytest.param( + {"input": "hi", "tools": [dict(_PATTERNED_TOOL)]}, + "native Responses does not implement tool schema repair", + id="python_incompatible_pattern", + ), + pytest.param( + {"input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}]}, + "native Responses does not implement prompt cache breakpoints", + id="prompt_cache_breakpoints_on_openai", + ), + pytest.param( + { + "input": [{"role": "user", "content": "hi", "cache_control": {"type": "ephemeral"}}], + "api_base": "http://127.0.0.1:9/v1", + }, + None, + id="cache_control_on_a_compatible_host", + ), + ), +) +def test_model_rules_stay_on_python(request_fields: dict[str, object], expected: str | None) -> None: + assert decline_reason(MappingProxyType({"model": "openai/gpt-6.1-sol", **request_fields})) == expected