diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 8d189c8c515..63b4c935208 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3426,6 +3426,7 @@ dependencies = [ "rstest", "serde", "serde_json", + "sha2 0.10.9", "thiserror 2.0.19", "tokio", ] diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs index 7a27519c269..2d94d766146 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -225,7 +225,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { } #[rstest] -#[case::content_parts_skip_images( +#[case::content_parts_keep_images_in_place( json!([ {"role": "user", "content": "hello"}, { @@ -237,9 +237,9 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { ], }, ]), - "helloworld!" + r#"helloworld{"type":"image_url","image_url":{"url":"ignored"}}!"# )] -#[case::search_results_and_compact_citations( +#[case::tool_message_with_search_results_kept_as_json( json!([{ "role": "tool", "content": null, @@ -250,7 +250,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { "citations": {"page": 1, "section": "intro"}, }], }]), - r#"{"result_of_call":null,"output":""}sourcetitlebody{"page":1,"section":"intro"}"# + r#"{"role":"tool","search_results":[{"source":"source","title":"title","content":[{"text":"body"}],"citations":{"page":1,"section":"intro"}}]}"# )] #[tokio::test(flavor = "multi_thread")] async fn prompt_matches_python_message_rules( diff --git a/litellm-rust/crates/cache/Cargo.toml b/litellm-rust/crates/cache/Cargo.toml index f18dbd9cb26..69b898adb11 100644 --- a/litellm-rust/crates/cache/Cargo.toml +++ b/litellm-rust/crates/cache/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [dependencies] serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } +sha2.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index 253efb47759..db14abee664 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -8,9 +8,10 @@ use std::{collections::HashMap, future::Future, io}; use serde::Serialize; use serde_json::{ - Value, json, + Map, Value, ser::{CharEscape, Formatter, Serializer}, }; +use sha2::{Digest, Sha256}; use crate::{BaseCache, Error, SemanticCacheContext}; @@ -83,122 +84,225 @@ impl Embedder for PreparedEmbedding { } } -/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results, -/// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the -/// position of the call it answers. +const PLAIN_TYPES: [&str; 4] = ["text", "input_text", "output_text", "message"]; +const PLAIN_KEYS: [&str; 5] = ["role", "type", "text", "content", "status"]; +const TOOL_ROLES: [&str; 2] = ["tool", "function"]; +const IGNORED_KEYS: [&str; 1] = ["cache_control"]; +const CALL_ID_KEYS: [&str; 4] = ["id", "call_id", "tool_use_id", "tool_call_id"]; +const OPAQUE_KEYS: [&str; 4] = ["data", "file_data", "signature", "encrypted_content"]; + +/// `get_str_from_messages_with_tools`: plain text parts stay plain text, and every other part is +/// kept as compact JSON with call ids replaced by their position and opaque blobs by a digest. pub fn str_from_messages(messages: &[Value]) -> String { - let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect(); - let call_ordinals = tool_call_ordinals(messages.iter().flat_map(|message| { - let block_ids = message - .get("content") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_use")) - .filter_map(|block| block.get("id")); - let tool_call_ids = message - .get("tool_calls") - .and_then(Value::as_array) - .into_iter() - .flatten() - .filter_map(|tool_call| tool_call.get("id")); - block_ids.chain(tool_call_ids) - })); - let mut text = String::new(); - for message in messages { - if message.get("role").and_then(Value::as_str) == Some("tool") { - let mut output = String::new(); - push_content_text(&mut output, message.get("content"), &call_ordinals); - text.push_str(&tool_result_json( - message.get("tool_call_id"), - &call_ordinals, - &output, - )); - } else { - push_content_text(&mut text, message.get("content"), &call_ordinals); - } - if let Some(Value::Array(tool_calls)) = message.get("tool_calls") { - for tool_call in tool_calls.iter().filter_map(Value::as_object) { - let function = tool_call.get("function"); - text.push_str(&tool_call_json( - function.and_then(|function| function.get("name")), - function.and_then(|function| function.get("arguments")), - )); - } - } - push_search_results_text(&mut text, message.get("search_results")); - } - text + let messages: Vec<&Value> = messages + .iter() + .filter(|message| message.is_object()) + .collect(); + let call_ordinals = call_id_ordinals(messages.iter().copied()); + messages + .into_iter() + .filter_map(Value::as_object) + .map(|message| message_prompt(message, &call_ordinals)) + .collect() } -/// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content. -fn push_content_text( - text: &mut String, +/// `get_str_from_responses_input`: `str_from_messages` for a Responses API `input`, one stripped +/// line per part. `None` when nothing is left. +fn str_from_responses_input(input: &Value) -> Option { + let call_ordinals = call_id_ordinals(std::iter::once(input)); + let mut parts = Vec::new(); + push_responses_input_parts(input, &call_ordinals, &mut parts); + let prompt = python_strip(&parts.join("\n")).to_owned(); + (!prompt.is_empty()).then_some(prompt) +} + +fn message_prompt(message: &Map, call_ordinals: &HashMap<&str, usize>) -> String { + let is_tool = message + .get("role") + .and_then(Value::as_str) + .is_some_and(|role| TOOL_ROLES.contains(&role)); + if is_tool || !is_plain(message) { + return compact_json(&normalized(message, call_ordinals)); + } + plain_prompt(message, call_ordinals) +} + +fn plain_prompt(value: &Map, call_ordinals: &HashMap<&str, usize>) -> String { + let mut prompt = value + .get("text") + .and_then(Value::as_str) + .unwrap_or_default() + .to_owned(); + push_content_prompt(&mut prompt, value.get("content"), call_ordinals); + prompt +} + +fn push_content_prompt( + prompt: &mut String, content: Option<&Value>, call_ordinals: &HashMap<&str, usize>, ) { match content { - Some(Value::String(content)) => text.push_str(content), - Some(Value::Array(blocks)) => { - for block in blocks.iter().filter_map(Value::as_object) { - match block.get("type").and_then(Value::as_str) { - Some("tool_use") => { - text.push_str(&tool_call_json(block.get("name"), block.get("input"))); - } - Some("tool_result") => { - let mut output = String::new(); - push_content_text(&mut output, block.get("content"), call_ordinals); - text.push_str(&tool_result_json( - block.get("tool_use_id"), - call_ordinals, - &output, - )); - } - _ => { - if let Some(block_text) = block.get("text").and_then(Value::as_str) { - text.push_str(block_text); - } - } - } + None | Some(Value::Null) => {} + Some(Value::String(text)) => prompt.push_str(text), + Some(Value::Array(items)) => { + for item in items { + push_content_prompt(prompt, Some(item), call_ordinals); } } - _ => {} + Some(Value::Object(map)) if is_plain(map) => { + prompt.push_str(&plain_prompt(map, call_ordinals)); + } + Some(Value::Object(map)) => prompt.push_str(&compact_json(&normalized(map, call_ordinals))), + Some(other) => prompt.push_str(&compact_json(other)), } } -/// `tool_call_ordinals`: the 1-based position of each distinct string call id, first seen first. -fn tool_call_ordinals<'a>(call_ids: impl Iterator) -> HashMap<&'a str, usize> { +fn push_responses_input_parts( + value: &Value, + call_ordinals: &HashMap<&str, usize>, + parts: &mut Vec, +) { + match value { + Value::Null => {} + Value::String(text) => push_stripped(text, parts), + Value::Array(items) => { + for item in items { + push_responses_input_parts(item, call_ordinals, parts); + } + } + Value::Object(map) if is_plain(map) => { + push_stripped( + map.get("text").and_then(Value::as_str).unwrap_or_default(), + parts, + ); + if let Some(content) = map.get("content") { + push_responses_input_parts(content, call_ordinals, parts); + } + } + Value::Object(map) => parts.push(compact_json(&normalized(map, call_ordinals))), + other => parts.push(compact_json(other)), + } +} + +fn push_stripped(text: &str, parts: &mut Vec) { + let stripped = python_strip(text); + if !stripped.is_empty() { + parts.push(stripped.to_owned()); + } +} + +/// `_is_plain`: a text-only part whose every kept key is a plain or call id key. +fn is_plain(value: &Map) -> bool { + let plain_type = match value.get("type") { + None | Some(Value::Null) => true, + Some(Value::String(value_type)) => PLAIN_TYPES.contains(&value_type.as_str()), + Some(_) => false, + }; + let plain_text = matches!( + value.get("text"), + None | Some(Value::Null | Value::String(_)) + ); + plain_type + && plain_text + && kept_entries(value).all(|(key, _)| { + PLAIN_KEYS.contains(&key.as_str()) || CALL_ID_KEYS.contains(&key.as_str()) + }) +} + +fn kept_entries(value: &Map) -> impl Iterator { + value + .iter() + .filter(|(key, item)| !IGNORED_KEYS.contains(&key.as_str()) && !is_empty(item)) +} + +fn is_empty(value: &Value) -> bool { + match value { + Value::Null => true, + Value::String(text) => text.is_empty(), + Value::Array(items) => items.is_empty(), + Value::Object(map) => map.is_empty(), + Value::Bool(_) | Value::Number(_) => false, + } +} + +/// `_normalized`: the part minus ignored and empty entries, with call ids replaced by their +/// position and opaque blobs by `sha256:` plus the first 16 hex digits of their digest. +fn normalized(value: &Map, call_ordinals: &HashMap<&str, usize>) -> Value { + Value::Object( + kept_entries(value) + .map(|(key, item)| { + ( + key.clone(), + normalized_value(item, call_ordinals, Some(key)), + ) + }) + .collect(), + ) +} + +fn normalized_value( + value: &Value, + call_ordinals: &HashMap<&str, usize>, + key: Option<&str>, +) -> Value { + match value { + Value::Object(map) => normalized(map, call_ordinals), + Value::Array(items) => Value::Array( + items + .iter() + .map(|item| normalized_value(item, call_ordinals, None)) + .collect(), + ), + Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => call_ordinals + .get(text.as_str()) + .map_or_else(|| value.clone(), |ordinal| Value::from(*ordinal)), + Value::String(text) + if key.is_some_and(|key| OPAQUE_KEYS.contains(&key)) || text.starts_with("data:") => + { + Value::String(digest(text)) + } + other => other.clone(), + } +} + +fn digest(text: &str) -> String { + let hash = Sha256::digest(text.as_bytes()); + let hex: String = hash[..8].iter().map(|byte| format!("{byte:02x}")).collect(); + format!("sha256:{hex}") +} + +/// `_call_id_ordinals`: the 1-based position of each distinct call id string, first seen first. +fn call_id_ordinals<'a>(values: impl Iterator) -> HashMap<&'a str, usize> { + let mut ids = Vec::new(); + for value in values { + collect_call_ids(value, None, &mut ids); + } let mut ordinals = HashMap::new(); - for call_id in call_ids.filter_map(Value::as_str) { + for id in ids { let next = ordinals.len() + 1; - ordinals.entry(call_id).or_insert(next); + ordinals.entry(id).or_insert(next); } ordinals } -/// `tool_result_str`: `{"result_of_call":N,"output":...}`, with a `null` position when the result -/// answers no known call. -fn tool_result_json( - call_id: Option<&Value>, - call_ordinals: &HashMap<&str, usize>, - output: &str, -) -> String { - let ordinal = call_id - .and_then(Value::as_str) - .and_then(|call_id| call_ordinals.get(call_id)); - format!( - "{{\"result_of_call\":{},\"output\":{}}}", - compact_json(&json!(ordinal)), - compact_json(&Value::String(output.to_owned())), - ) -} - -/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes. -fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String { - compact_json(&json!({ - "name": name.unwrap_or(&Value::Null), - "arguments": arguments.unwrap_or(&Value::Null), - })) +fn collect_call_ids<'a>(value: &'a Value, key: Option<&str>, ids: &mut Vec<&'a str>) { + match value { + Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => ids.push(text), + Value::Object(map) => { + for (child_key, child) in map { + if !IGNORED_KEYS.contains(&child_key.as_str()) { + collect_call_ids(child, Some(child_key), ids); + } + } + } + Value::Array(items) => { + for item in items { + collect_call_ids(item, None, ids); + } + } + _ => {} + } } /// The messages prompt Qdrant embeds: `None` when the request carries no messages. @@ -215,103 +319,7 @@ pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { { return Some(str_from_messages(messages)); } - let input = context.input.as_ref()?; - let call_ordinals = tool_call_ordinals( - input - .as_array() - .into_iter() - .flatten() - .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call")) - .filter_map(|item| item.get("call_id")), - ); - let mut parts = Vec::new(); - collect_input_text(input, &mut parts, &call_ordinals); - let prompt = python_strip(&parts.join("\n")).to_owned(); - (!prompt.is_empty()).then_some(prompt) -} - -/// `extract_search_results_text`. -fn push_search_results_text(text: &mut String, search_results: Option<&Value>) { - let Some(Value::Array(results)) = search_results else { - return; - }; - for result in results.iter().filter_map(Value::as_object) { - for key in ["source", "title"] { - if let Some(value) = result.get(key).and_then(Value::as_str) { - text.push_str(value); - } - } - if let Some(Value::Array(content)) = result.get("content") { - for block in content.iter().filter_map(Value::as_object) { - if let Some(value) = block.get("text").and_then(Value::as_str) { - text.push_str(value); - } - } - } - if let Some(citations) = result.get("citations").filter(|value| !value.is_null()) { - text.push_str(&compact_json(citations)); - } - } -} - -fn collect_input_text( - value: &Value, - parts: &mut Vec, - call_ordinals: &HashMap<&str, usize>, -) { - match value { - Value::String(text) => { - push_trimmed(text, parts); - } - Value::Array(items) => { - for item in items { - collect_input_text(item, parts, call_ordinals); - } - } - Value::Object(map) => { - if map.get("type").and_then(Value::as_str) == Some("function_call") { - parts.push(tool_call_json(map.get("name"), map.get("arguments"))); - return; - } - if map.get("type").and_then(Value::as_str) == Some("function_call_output") { - let mut output_parts = Vec::new(); - if let Some(output) = map.get("output") { - collect_input_text(output, &mut output_parts, call_ordinals); - } - parts.push(tool_result_json( - map.get("call_id"), - call_ordinals, - python_strip(&output_parts.join("\n")), - )); - return; - } - if let Some(content) = map.get("content").filter(|content| !content.is_null()) { - collect_input_text(content, parts, call_ordinals); - return; - } - for key in ["text", "output", "input_text", "output_text"] { - match map.get(key) { - Some(nested @ Value::Array(_)) => { - collect_input_text(nested, parts, call_ordinals); - return; - } - Some(Value::String(text)) if push_trimmed(text, parts) => return, - _ => {} - } - } - } - _ => {} - } -} - -/// Pushes `text` stripped as Python's `str.strip` does, reporting whether anything was left. -fn push_trimmed(text: &str, parts: &mut Vec) -> bool { - let trimmed = python_strip(text); - if trimmed.is_empty() { - return false; - } - parts.push(trimmed.to_owned()); - true + str_from_responses_input(context.input.as_ref()?) } /// `str.strip()`: Python's whitespace also covers the ASCII information separators. diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index af87e6e0164..c4a1bfb2d56 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -16,168 +16,115 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex } #[rstest] -#[case::empty(json!([]), "")] -#[case::string_content(json!([{"role": "user", "content": "hello"}]), "hello")] +#[case::empty(r#"[]"#, r#""#)] +#[case::string_content(r#"[{"role": "user", "content": "hello"}]"#, r#"hello"#)] #[case::concatenates_messages( - json!([{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]), - "be brief. hello", + r#"[{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]"#, + r#"be brief. hello"# )] #[case::text_parts( - json!([{"role": "user", "content": [ - {"type": "text", "text": "What is "}, - {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, - {"type": "text", "text": "this?"}, - ]}]), - "What is this?", + r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "text", "text": "this?"}]}]"#, + r#"What is this?"#, +)] +#[case::image_part_kept_in_place( + r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "this?"}]}]"#, + r#"What is {"type":"image_url","image_url":{"url":"https://example.com/a.png"}}this?"#, )] #[case::missing_null_and_empty_content( - json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]), - "", + r#"[{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]"#, + r#""#, )] -#[case::search_results_hidden_behind_small_content( - json!([{"role": "tool", "content": "small", "search_results": [ - {"source": "s", "title": "t", "content": [{"text": "hidden payload"}]}, - ]}]), - r#"{"result_of_call":null,"output":"small"}sthidden payload"#, +#[case::non_object_messages_skipped( + r#"["junk", 3, {"role": "user", "content": "kept"}]"#, + r#"kept"# )] -#[case::title_only_search_result( - json!([{"role": "tool", "content": "small", "search_results": [ - {"source": "s", "title": "long title", "content": []}, - ]}]), - r#"{"result_of_call":null,"output":"small"}slong title"#, +#[case::cache_control_ignored( + r#"[{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}]"#, + r#"hi"#, )] -#[case::search_results_without_content( - json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]), - r#"{"result_of_call":null,"output":""}st"#, +#[case::extra_message_field_kept( + r#"[{"role": "user", "name": "alice", "content": "hi"}]"#, + r#"{"role":"user","name":"alice","content":"hi"}"# )] -#[case::search_result_fields_in_python_order( - json!([{"role": "tool", "content": "c", "search_results": [ - {"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"}, - {"source": "s2"}, - ]}]), - r#"{"result_of_call":null,"output":"c"}stbody{"enabled":true}s2"#, +#[case::unknown_block_type_kept( + r#"[{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}]"#, + r#"{"type":"brand_new_block","payload":{"k":"v"}}!"#, )] -#[case::null_citations_skipped( - json!([{"role": "tool", "content": "c", "search_results": [ - {"source": "s", "citations": null}, - ]}]), - r#"{"result_of_call":null,"output":"c"}s"#, +#[case::non_string_text_kept( + r#"[{"role": "user", "content": [{"type": "text", "text": 3}]}]"#, + r#"{"type":"text","text":3}"# )] -#[case::non_string_and_non_object_entries_skipped( - json!([{"role": "tool", "content": "c", "search_results": [ - "junk", - {"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]}, - ]}]), - r#"{"result_of_call":null,"output":"c"}kept"#, +#[case::scalar_content_kept( + r#"[{"role": "user", "content": 7}, {"role": "user", "content": [1.5, true, "x"]}]"#, + r#"71.5truex"# )] -#[case::non_list_search_results_skipped( - json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]), - r#"{"result_of_call":null,"output":"c"}"#, +#[case::search_results_kept( + r#"[{"role": "tool", "content": "small", "search_results": [{"source": "s", "title": "t", "content": [{"type": "text", "text": "hidden payload"}], "citations": {"enabled": true}}]}]"#, + r#"{"role":"tool","content":"small","search_results":[{"source":"s","title":"t","content":[{"type":"text","text":"hidden payload"}],"citations":{"enabled":true}}]}"#, )] -#[case::citations_compact_in_insertion_order( - json!([{"role": "tool", "search_results": [ - {"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}}, - ]}]), - r#"{"result_of_call":null,"output":""}{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, -)] -#[case::citations_ensure_ascii( - json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]), - r#"{"result_of_call":null,"output":""}["caf\u00e9","\u4e2d"]"#, -)] -#[case::citations_astral_chars_as_surrogate_pairs( - json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]), - r#"{"result_of_call":null,"output":""}"\ud83d\ude00""#, -)] -#[case::citations_escapes( - json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]), - r#"{"result_of_call":null,"output":""}"q\"\\\n\t\u0001/""#, -)] -#[case::citations_large_float_exponent( - json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]), - r#"{"result_of_call":null,"output":""}[1e+20,1.0]"#, -)] -#[case::citations_scalars( - json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]), - r#"{"result_of_call":null,"output":""}false3"#, -)] -#[case::anthropic_tool_use_name_and_input_without_id( - json!([ - {"role": "user", "content": "fix the failing test"}, - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}, - ]}, - ]), - r#"fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}"#, +#[case::anthropic_tool_use( + r#"[{"role": "user", "content": "fix the failing test"}, {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}]}]"#, + r#"fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}"#, )] #[case::anthropic_string_tool_result( - json!([{"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}, - ]}]), - r#"{"result_of_call":null,"output":"calc.py"}"#, + r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}]"#, + r#"{"type":"tool_result","tool_use_id":1,"content":"calc.py"}"#, )] #[case::anthropic_nested_text_tool_result_then_text( - json!([{"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t1", "content": [ - {"type": "text", "text": "a"}, - {"type": "text", "text": "b"}, - ]}, - {"type": "text", "text": "next"}, - ]}]), - r#"{"result_of_call":null,"output":"ab"}next"#, + r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}, {"type": "text", "text": "next"}]}]"#, + r#"{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next"#, )] -#[case::openai_tool_calls_in_order_before_tool_result( - json!([ - {"role": "assistant", "content": "writing", "tool_calls": [ - {"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, - {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}, - ]}, - {"role": "tool", "tool_call_id": "c1", "content": "ok"}, - ]), - r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1,"output":"ok"}"#, +#[case::openai_tool_calls_then_tool_result( + r#"[{"role": "assistant", "content": "writing", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}]}, {"role": "tool", "tool_call_id": "c1", "content": "ok"}]"#, + r#"{"role":"assistant","content":"writing","tool_calls":[{"id":1,"type":"function","function":{"name":"write","arguments":"{\"path\": \"a\"}"}},{"id":2,"type":"function","function":{"name":"write","arguments":"{\"path\": \"b\"}"}}]}{"role":"tool","tool_call_id":1,"content":"ok"}"#, )] #[case::anthropic_parallel_tool_results_tagged_with_their_call( - json!([ - {"role": "assistant", "content": [ - {"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, - {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}, - ]}, - {"role": "user", "content": [ - {"type": "tool_result", "tool_use_id": "t2", "content": "B"}, - {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, - ]}, - ]), - r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}"#, + r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t2", "content": "B"}, {"type": "tool_result", "tool_use_id": "t1", "content": "A"}]}]"#, + r#"{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}"#, )] #[case::reused_call_ids_keep_first_position( - json!([ - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}, - ]}, - {"role": "assistant", "content": null, "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, - {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, - ]}, - {"role": "tool", "tool_call_id": "call_1", "content": "C"}, - ]), - r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2,"output":"C"}"#, + r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}]}, {"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}]}, {"role": "tool", "tool_call_id": "call_1", "content": "C"}]"#, + r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"b","arguments":"{}"}},{"id":2,"type":"function","function":{"name":"c","arguments":"{}"}}]}{"role":"tool","tool_call_id":2,"content":"C"}"#, )] -#[case::unknown_call_ids_encode_a_null_position( - json!([ - {"role": "tool", "tool_call_id": "c9", "content": "ok"}, - {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]}, - ]), - r#"{"result_of_call":null,"output":"ok"}{"result_of_call":null,"output":"done"}"#, -)] -#[case::tool_output_cannot_forge_an_encoded_result( - json!([{"role": "tool", "tool_call_id": "c1", "content": "\"},{\"result_of_call\":2,\"output\":\""}]), - r#"{"result_of_call":null,"output":"\"},{\"result_of_call\":2,\"output\":\""}"#, +#[case::tool_output_cannot_forge_a_record( + r#"[{"role": "tool", "content": "x\"}{\"role\":\"tool\",\"content\":\"y"}]"#, + r#"{"role":"tool","content":"x\"}{\"role\":\"tool\",\"content\":\"y"}"# )] #[case::malformed_tool_call_entries( - json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]), - r#"{"name":null,"arguments":null}"#, + r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]"#, + r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function"},"junk"]}"#, )] -fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) { - assert_eq!(str_from_messages(messages.as_array().unwrap()), expected); +#[case::anthropic_base64_image_digested( + r#"[{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#, + r#"{"type":"image","source":{"type":"base64","media_type":"image/png","data":"sha256:b1a66f8de3276946"}}"#, +)] +#[case::openai_data_url_digested( + r#"[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#, + r#"{"type":"image_url","image_url":{"url":"sha256:19c7f028a60a33b8"}}"#, +)] +#[case::thinking_signature_digested( + r#"[{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": "EqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARAB"}, {"type": "text", "text": "done"}]}]"#, + r#"{"type":"thinking","thinking":"hmm","signature":"sha256:b20cbbd82e4b3791"}done"#, +)] +#[case::non_ascii_escaped( + r#"[{"role": "tool", "content": "caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}]"#, + r#"{"role":"tool","content":"caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}"# +)] +#[case::non_ascii_plain_text_kept_raw( + r#"[{"role": "user", "content": "caf\u00e9 \ud83d\ude00"}]"#, + r#"café 😀"# +)] +#[case::floats_in_python_repr( + r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {"a": [1e+20, 1.0, 1.5e-07, 0.0001, 1e+16, -2.5, 3]}}]}]"#, + r#"{"type":"tool_use","id":1,"name":"n","input":{"a":[1e+20,1.0,1.5e-07,0.0001,1e+16,-2.5,3]}}"#, +)] +#[case::empty_fields_dropped( + r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {}, "extra": [], "note": ""}]}]"#, + r#"{"type":"tool_use","id":1,"name":"n"}"#, +)] +fn str_from_messages_matches_python(#[case] messages: &str, #[case] expected: &str) { + let messages: Vec = serde_json::from_str(messages).unwrap(); + assert_eq!(str_from_messages(&messages), expected); } #[rstest] @@ -196,167 +143,109 @@ fn prompt_from_messages_reads_messages_only( #[rstest] #[case::prefers_messages( - Some(json!([{"content": "message prompt"}])), - Some(json!("responses prompt")), - Some("message prompt"), + Some(r#"[{"content": "message prompt"}]"#), + Some(r#""responses prompt""#), + Some(r#"message prompt"#) )] #[case::empty_messages_fall_back_to_input( - Some(json!([])), - Some(json!("responses prompt")), - Some("responses prompt"), + Some(r#"[]"#), + Some(r#""responses prompt""#), + Some(r#"responses prompt"#) )] #[case::messages_without_text_keep_an_empty_prompt( - Some(json!([{"content": null}])), - Some(json!("x")), - Some(""), + Some(r#"[{"content": null}]"#), + Some(r#""x""#), + Some(r#""#) )] -#[case::nothing(None, None, None)] -#[case::null_input(None, Some(Value::Null), None)] -#[case::blank_string(None, Some(json!(" ")), None)] +#[case::null_input(None, None, None)] +#[case::blank_string(None, Some(r#"" ""#), None)] #[case::trimmed_string( None, - Some(json!(" What is the capital of France?\n")), - Some("What is the capital of France?"), + Some(r#"" What is the capital of France?\n""#), + Some(r#"What is the capital of France?"#) )] -#[case::image_only( - None, - Some(json!([{"type": "input_image", "image_url": "https://example.com"}])), +#[case::strip_information_separators(None, Some(r#""\u001ca\u001f""#), Some(r#"a"#))] +#[case::image_only_kept( None, + Some(r#"[{"type": "input_image", "image_url": "https://example.com"}]"#), + Some(r#"{"type":"input_image","image_url":"https://example.com"}"#) )] #[case::structured_input( None, - Some(json!([{"role": "user", "content": [ - {"type": "input_text", "text": "What is the capital of France?"}, - {"type": "input_text", "text": "Answer briefly."}, - {"type": "input_image", "image_url": "https://example.com/paris.png"}, - ]}])), - Some("What is the capital of France?\nAnswer briefly."), + Some(r#"[{"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}, {"type": "input_text", "text": "Answer briefly."}, {"type": "input_image", "image_url": "https://example.com/paris.png"}]}]"#), + Some(r#"What is the capital of France? +Answer briefly. +{"type":"input_image","image_url":"https://example.com/paris.png"}"#), )] -#[case::model_objects_after_dump( +#[case::message_items_with_ids( None, - Some(json!([ - {"content": [{"text": "model dump prompt"}]}, - {"content": [{"output_text": "dict prompt"}]}, - {"content": [{"input_text": "inline prompt"}]}, - {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, - ])), - Some("model dump prompt\ndict prompt\ninline prompt"), + Some(r#"[{"type": "message", "id": "msg_1", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "hi", "annotations": []}]}]"#), + Some(r#"hi"#), )] #[case::object_content( None, - Some(json!({"content": [{"text": "object content prompt"}]})), - Some("object content prompt"), + Some(r#"{"content": [{"text": "object content prompt"}]}"#), + Some(r#"object content prompt"#) )] -#[case::string_content(None, Some(json!({"content": " inline "})), Some("inline"))] -#[case::null_content_uses_text_keys( +#[case::string_content(None, Some(r#"{"content": " inline "}"#), Some(r#"inline"#))] +#[case::unknown_keys_kept( None, - Some(json!({"content": null, "output": "tool output"})), - Some("tool output"), + Some(r#"{"text": " ", "input_text": "fallback prompt"}"#), + Some(r#"{"text":" ","input_text":"fallback prompt"}"#) )] -#[case::content_wins_over_text(None, Some(json!({"content": [], "text": "ignored"})), None)] -#[case::text_key_precedence( +#[case::nested_lists( None, - Some(json!({"output_text": "d", "input_text": "c", "output": "b", "text": "a"})), - Some("a"), + Some(r#"[["a", [" b "]], "", "c"]"#), + Some( + r#"a +b +c"# + ) )] -#[case::input_text_key(None, Some(json!({"input_text": "only input"})), Some("only input"))] -#[case::output_text_key(None, Some(json!({"output_text": "only output"})), Some("only output"))] -#[case::non_string_text_keys_skipped( +#[case::scalars_kept( None, - Some(json!({"text": 1, "output": "fallback"})), - Some("fallback"), + Some(r#"[1, true, null, "kept"]"#), + Some( + r#"1 +true +kept"# + ) )] -#[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))] -#[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))] +#[case::blank_text_parts(None, Some(r#"[{"type": "input_text", "text": " "}]"#), None)] #[case::responses_function_call( None, - Some(json!([ - {"role": "user", "content": "update the config"}, - {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, - {"type": "function_call_output", "call_id": "c1", "output": "ok"}, - ])), - Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1,\"output\":\"ok\"}"), + Some(r#"[{"role": "user", "content": "update the config"}, {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, {"type": "function_call_output", "call_id": "c1", "output": "ok"}]"#), + Some(r#"update the config +{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\"path\":\"a.yaml\"}"} +{"type":"function_call_output","call_id":1,"output":"ok"}"#), )] #[case::responses_structured_function_call_output( None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]}, - ])), - Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1,\"output\":\"denied\"}"), -)] -#[case::responses_multi_part_output_joined_by_lines( - None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, - {"type": "function_call_output", "call_id": "c1", "output": [ - {"type": "input_text", "text": " line one "}, - {"type": "input_text", "text": "line two"}, - ]}, - ])), - Some(r#"{"name":"run","arguments":"{}"} -{"result_of_call":1,"output":"line one\nline two"}"#), + Some(r#"[{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": " denied "}]}]"#), + Some(r#"{"type":"function_call","call_id":1,"name":"write_file","arguments":"{}"} +{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":" denied "}]}"#), )] #[case::responses_parallel_outputs_tagged_with_their_call( None, - Some(json!([ - {"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, - {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, - {"type": "function_call_output", "call_id": "c2", "output": "B"}, - {"type": "function_call_output", "call_id": "c1", "output": "A"}, - ])), - Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2,\"output\":\"B\"}\n{\"result_of_call\":1,\"output\":\"A\"}"), + Some(r#"[{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, {"type": "function_call_output", "call_id": "c2", "output": "B"}, {"type": "function_call_output", "call_id": "c1", "output": "A"}]"#), + Some(r#"{"type":"function_call","call_id":1,"name":"read","arguments":"a"} +{"type":"function_call","call_id":2,"name":"read","arguments":"b"} +{"type":"function_call_output","call_id":2,"output":"B"} +{"type":"function_call_output","call_id":1,"output":"A"}"#), )] -#[case::responses_unknown_call_id_encodes_a_null_position( +#[case::responses_reasoning_digested( None, - Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])), - Some(r#"{"result_of_call":null,"output":"orphan"}"#), + Some(r#"[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "gAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAAB"}]"#), + Some(r#"{"type":"reasoning","id":1,"encrypted_content":"sha256:010b9eb723d8d1e2"}"#), )] fn prompt_from_context_matches_python( - #[case] messages: Option, - #[case] input: Option, + #[case] messages: Option<&str>, + #[case] input: Option<&str>, #[case] expected: Option<&str>, ) { - assert_eq!( - prompt_from_context(&context(messages, input)).as_deref(), - expected - ); -} - -/// Python `test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys`: a blank -/// text key falls through to the next one. -#[rstest] -#[case::blank_text_falls_through( - json!({"text": " ", "input_text": "fallback prompt"}), - "fallback prompt", -)] -fn prompt_from_context_skips_blank_text_keys(#[case] input: Value, #[case] expected: &str) { - assert_eq!( - prompt_from_context(&context(None, Some(input))).as_deref(), - Some(expected) - ); -} - -/// Where `json.dumps(..., separators=(",", ":"))` and Python `str.strip` differ from -/// `semantic.rs`: ensure_ascii escapes DEL, small floats keep Python's two-digit exponent, and -/// strip also removes the ASCII information separators. -#[rstest] -#[case::del_is_escaped(json!([{"search_results": [{"citations": "\u{7f}"}]}]), None, r#""\u007f""#)] -#[case::small_float_exponent(json!([{"search_results": [{"citations": 1.5e-7}]}]), None, "1.5e-07")] -#[case::float_at_positional_floor(json!([{"search_results": [{"citations": 1e-4}]}]), None, "0.0001")] -#[case::float_at_scientific_ceiling(json!([{"search_results": [{"citations": 1e16}]}]), None, "1e+16")] -#[case::large_float(json!([{"search_results": [{"citations": [1.25e20, -2.5, 3.0]}]}]), None, "[1.25e+20,-2.5,3.0]")] -#[case::strip_information_separators(json!([]), Some(json!("\u{1c}a\u{1f}")), "a")] -fn python_serialization_edge_cases( - #[case] messages: Value, - #[case] input: Option, - #[case] expected: &str, -) { - let actual = match input { - Some(input) => prompt_from_context(&context(None, Some(input))).unwrap_or_default(), - None => str_from_messages(messages.as_array().unwrap()), - }; - assert_eq!(actual, expected); + let parse = |value: &str| serde_json::from_str::(value).unwrap(); + let context = context(messages.map(parse), input.map(parse)); + assert_eq!(prompt_from_context(&context).as_deref(), expected); } #[rstest] diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 827fa91398f..9bf88989958 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -22,9 +22,7 @@ from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages_with_tools, - tool_call_ordinals, - tool_call_str, - tool_result_str, + get_str_from_responses_input, ) from litellm.types.utils import EmbeddingResponse @@ -271,101 +269,7 @@ class RedisSemanticCache(BaseCache): if "input" not in kwargs: return None - responses_input: Final = kwargs.get("input") - prompt: Final = cls._responses_input_prompt(responses_input, cls._responses_call_ordinals(responses_input)) - return prompt or None - - @classmethod - def _responses_input_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> str: - prompt_parts: Final[list[str]] = [] - cls._collect_responses_input_text(value, prompt_parts, call_ordinals) - return "\n".join(prompt_parts).strip() - - @classmethod - def _collect_responses_input_text( - cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int] - ) -> None: - value = cls._function_call_as_prompt(cls._coerce_response_input_value(value), call_ordinals) - if value is None: - return - - if isinstance(value, str): - stripped_value: Final = value.strip() - if stripped_value: - prompt_parts.append(stripped_value) - return - - if isinstance(value, (list, tuple)): - for item in value: - cls._collect_responses_input_text(item, prompt_parts, call_ordinals) - return - - if isinstance(value, dict): - content = value.get("content") - if content is not None: - cls._collect_responses_input_text(content, prompt_parts, call_ordinals) - return - - cls._collect_responses_text_fields(value, prompt_parts, call_ordinals) - return - - content = getattr(value, "content", None) - if content is not None: - cls._collect_responses_input_text(content, prompt_parts, call_ordinals) - return - - for text_key in ("text", "output", "input_text", "output_text"): - text_value = getattr(value, text_key, None) - if isinstance(text_value, str): - stripped_text = text_value.strip() - if stripped_text: - prompt_parts.append(stripped_text) - return - - @classmethod - def _collect_responses_text_fields( - cls, value: dict, prompt_parts: list[str], call_ordinals: Mapping[str, int] - ) -> None: - for text_key in ("text", "output", "input_text", "output_text"): - text_value = value.get(text_key) - if isinstance(text_value, (list, tuple)): - cls._collect_responses_input_text(text_value, prompt_parts, call_ordinals) - return - if isinstance(text_value, str) and (stripped_text := text_value.strip()): - prompt_parts.append(stripped_text) - return - - @classmethod - def _responses_call_ordinals(cls, responses_input: object) -> Mapping[str, int]: - items: Final = responses_input if isinstance(responses_input, (list, tuple)) else () - dumped_items: Final = (cls._coerce_response_input_value(item) for item in items) - return tool_call_ordinals( - item.get("call_id") - for item in dumped_items - if isinstance(item, dict) and item.get("type") == "function_call" - ) - - @classmethod - def _function_call_as_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> object: - if not isinstance(value, dict): - return value - if value.get("type") == "function_call": - return tool_call_str(value.get("name"), value.get("arguments")) - if value.get("type") != "function_call_output": - return value - return tool_result_str( - value.get("call_id"), call_ordinals, cls._responses_input_prompt(value.get("output"), call_ordinals) - ) - - @staticmethod - def _coerce_response_input_value(value: object) -> object: - model_dump: Final = getattr(value, "model_dump", None) - if callable(model_dump): - return model_dump() - dict_method: Final = getattr(value, "dict", None) - if callable(dict_method): - return dict_method() - return value + return get_str_from_responses_input(kwargs.get("input")) def _embedding_input(self, prompt: str, router: "Router | None") -> str: return truncate_embedding_input( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 0c52fae86d7..8e75ac4034e 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2,6 +2,7 @@ Common utility functions used for translating messages across providers """ +import hashlib import io import json import mimetypes @@ -195,93 +196,148 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: return text +_PLAIN_TYPES: Final = frozenset({"text", "input_text", "output_text", "message"}) +_PLAIN_KEYS: Final = frozenset({"role", "type", "text", "content", "status"}) +_TOOL_ROLES: Final = frozenset({"tool", "function"}) +_IGNORED_KEYS: Final = frozenset({"cache_control"}) +_CALL_ID_KEYS: Final = frozenset({"id", "call_id", "tool_use_id", "tool_call_id"}) +_OPAQUE_KEYS: Final = frozenset({"data", "file_data", "signature", "encrypted_content"}) + + def get_str_from_messages_with_tools(messages: object) -> str: """ - ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns - that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) - produce different text. Each result is encoded with the position of the call it answers, since call ids are - random per session + Semantic-cache prompt for chat-style ``messages``. Plain text parts stay plain text so text-only prompts embed + as they always have. Every other part (tool calls and results, images, unknown block types, extra fields) is kept + as compact JSON, so a new request shape changes the prompt instead of silently vanishing from it """ message_mappings: Final = tuple(_str_mappings(messages)) - call_ordinals: Final = tool_call_ordinals(_message_tool_call_ids(message_mappings)) - return "".join(_message_str_with_tools(message, call_ordinals) for message in message_mappings) + call_ordinals: Final = _call_id_ordinals(message_mappings) + return "".join(_message_prompt(message, call_ordinals) for message in message_mappings) -def tool_call_str(name: object, arguments: object) -> str: - return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}' +def get_str_from_responses_input(responses_input: object) -> str | None: + """``get_str_from_messages_with_tools`` for Responses API ``input``, one stripped line per part""" + call_ordinals: Final = _call_id_ordinals(responses_input) + prompt: Final = "\n".join(_responses_input_parts(responses_input, call_ordinals)).strip() + return prompt or None -def tool_result_str(call_id: object, call_ordinals: Mapping[str, int], output: str) -> str: - ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None - return f'{{"result_of_call":{_compact_json(ordinal)},"output":{_compact_json(output)}}}' +def _message_prompt(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + role: Final = message.get("role") + if (isinstance(role, str) and role in _TOOL_ROLES) or not _is_plain(message): + return _compact_json(_normalized(message, call_ordinals)) + return _plain_prompt(message, call_ordinals) -def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]: - string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str)) - return MappingProxyType({call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(string_ids), start=1)}) +def _plain_prompt(value: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + text: Final = value.get("text") + return (text if isinstance(text, str) else "") + _content_prompt(value.get("content"), call_ordinals) -def _compact_json(value: object) -> str: - return json.dumps(value, separators=(",", ":"), default=str) +def _content_prompt(content: object, call_ordinals: Mapping[str, int]) -> str: + value: Final = _plain_value(content) + if value is None: + return "" + if isinstance(value, str): + return value + if isinstance(value, (list, tuple)): + return "".join(_content_prompt(item, call_ordinals) for item in value) + if _is_str_mapping(value) and _is_plain(value): + return _plain_prompt(value, call_ordinals) + return _compact_json(_normalized(value, call_ordinals)) -def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: - for message in messages: - yield from ( - block.get("id") for block in _str_mappings(message.get("content")) if block.get("type") == "tool_use" - ) - yield from (tool_call.get("id") for tool_call in _str_mappings(message.get("tool_calls"))) +def _responses_input_parts(value: object, call_ordinals: Mapping[str, int]) -> Iterator[str]: + item: Final = _plain_value(value) + if item is None: + return + if isinstance(item, str): + yield from _stripped(item) + elif isinstance(item, (list, tuple)): + for child in item: + yield from _responses_input_parts(child, call_ordinals) + elif _is_str_mapping(item) and _is_plain(item): + text: Final = item.get("text") + yield from _stripped(text if isinstance(text, str) else "") + yield from _responses_input_parts(item.get("content"), call_ordinals) + else: + yield _compact_json(_normalized(item, call_ordinals)) -def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - content: Final = _content_str_with_tools(message.get("content"), call_ordinals) +def _stripped(text: str) -> Iterator[str]: + stripped: Final = text.strip() + if stripped: + yield stripped + + +def _is_plain(value: Mapping[str, object]) -> bool: + value_type: Final = value.get("type") + text: Final = value.get("text") return ( - ( - tool_result_str(message.get("tool_call_id"), call_ordinals, content) - if message.get("role") == "tool" - else content - ) - + "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls"))) - + extract_search_results_text(message.get("search_results")) + (value_type is None or (isinstance(value_type, str) and value_type in _PLAIN_TYPES)) + and (text is None or isinstance(text, str)) + and all(key in _PLAIN_KEYS or key in _CALL_ID_KEYS for key in _kept_keys(value)) ) -def _content_str_with_tools(content: object, call_ordinals: Mapping[str, int]) -> str: - if isinstance(content, str): - return content - return "".join(_block_str_with_tools(block, call_ordinals) for block in _str_mappings(content)) +def _kept_keys(value: Mapping[str, object]) -> Iterator[str]: + return (key for key, item in value.items() if key not in _IGNORED_KEYS and not _is_empty(item)) -def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - block_type: Final = block.get("type") - if block_type == "tool_use": - return tool_call_str(block.get("name"), block.get("input")) - if block_type == "tool_result": - return tool_result_str( - block.get("tool_use_id"), call_ordinals, _content_str_with_tools(block.get("content"), call_ordinals) - ) - text: Final = block.get("text") - return text if isinstance(text, str) else "" +def _normalized(value: object, call_ordinals: Mapping[str, int], key: str | None = None) -> object: + item: Final = _plain_value(value) + if _is_str_mapping(item): + return MappingProxyType({kept: _normalized(item[kept], call_ordinals, kept) for kept in _kept_keys(item)}) + if isinstance(item, (list, tuple)): + return tuple(_normalized(child, call_ordinals) for child in item) + if not isinstance(item, str): + return item + if key in _CALL_ID_KEYS: + return call_ordinals.get(item, item) + if key in _OPAQUE_KEYS or item.startswith("data:"): + return f"sha256:{hashlib.sha256(item.encode('utf-8', 'surrogatepass')).hexdigest()[:16]}" + return item -def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str: - function: Final = _as_str_mapping(tool_call.get("function")) - if function is None: - return tool_call_str(None, None) - return tool_call_str(function.get("name"), function.get("arguments")) +def _call_id_ordinals(value: object) -> Mapping[str, int]: + return MappingProxyType( + {call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(_call_ids(value)), start=1)} + ) + + +def _call_ids(value: object, key: str | None = None) -> Iterator[str]: + item: Final = _plain_value(value) + if isinstance(item, str): + if key in _CALL_ID_KEYS: + yield item + elif _is_str_mapping(item): + for child_key, child in item.items(): + if child_key not in _IGNORED_KEYS: + yield from _call_ids(child, child_key) + elif isinstance(item, (list, tuple)): + for child in item: + yield from _call_ids(child) + + +def _is_empty(value: object) -> bool: + return value is None or (isinstance(value, (str, list, tuple, Mapping)) and not value) + + +def _compact_json(value: object) -> str: + return json.dumps(value, separators=(",", ":"), default=_json_default) + + +def _json_default(value: object) -> object: + return dict(value) if isinstance(value, MappingProxyType) else str(value) + + +def _plain_value(value: object) -> object: + return value.model_dump() if isinstance(value, BaseModel) else value def _str_mappings(values: object) -> Iterator[Mapping[str, object]]: items: Final = values if isinstance(values, (list, tuple)) else () - return (mapping for item in items if (mapping := _as_str_mapping(item)) is not None) - - -def _as_str_mapping(value: object) -> Mapping[str, object] | None: - if isinstance(value, BaseModel): - return value.model_dump() - if _is_str_mapping(value): - return value - return None + return (mapping for item in items if _is_str_mapping(mapping := _plain_value(item))) def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: message and block keys are str diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 3359796ec84..6040ae2b214 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -531,7 +531,7 @@ def test_redis_semantic_cache_get_cache_uses_responses_string_input(): ) -def test_redis_semantic_cache_set_cache_flattens_structured_responses_input(): +def test_redis_semantic_cache_set_cache_embeds_every_part_of_structured_responses_input(): from litellm.caching.redis_semantic_cache import RedisSemanticCache redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) @@ -561,7 +561,7 @@ def test_redis_semantic_cache_set_cache_flattens_structured_responses_input(): ) redis_semantic_cache.llmcache.store.assert_called_once_with( - "What is the capital of France?\nAnswer briefly.", + 'What is the capital of France?\nAnswer briefly.\n{"type":"input_image","image_url":"https://example.com/paris.png"}', "{'content': 'Paris'}", vector=[0.1, 0.2, 0.3], filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}, @@ -593,10 +593,11 @@ def test_redis_semantic_cache_prompt_extraction_keeps_tool_turns_distinct(): ] assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1,"output":"ok"}' + 'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}' + '{"type":"tool_result","tool_use_id":1,"content":"ok"}' ) - assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == ( - 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1,"output":"ok"}' + assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) != ( + RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) ) @@ -612,7 +613,8 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls() ) assert prompt == ( - 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1,"output":"ok"}' + 'update the config\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}' + '\n{"type":"function_call_output","call_id":1,"output":"ok"}' ) @@ -632,30 +634,13 @@ def test_redis_semantic_cache_prompt_extraction_keeps_structured_function_call_o ] ) - expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}' - assert ( - prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1,"output":"wrote 5 bytes"}}' + assert prompt_for("wrote 5 bytes") == ( + 'write hello\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}\n' + '{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":"wrote 5 bytes"}]}' ) assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError") -def test_redis_semantic_cache_prompt_extraction_joins_multi_part_function_call_output_lines(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - prompt = RedisSemanticCache._get_prompt_from_kwargs( - input=[ - {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, - { - "type": "function_call_output", - "call_id": "c1", - "output": [{"type": "input_text", "text": " line one "}, {"type": "input_text", "text": "line two"}], - }, - ] - ) - - assert prompt == '{"name":"run","arguments":"{}"}\n{"result_of_call":1,"output":"line one\\nline two"}' - - def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls(): from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -670,80 +655,63 @@ def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_ans ) assert prompt_for("c2", "c1") == ( - '{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n' - '{"result_of_call":2,"output":"empty"}\n{"result_of_call":1,"output":"secret"}' + '{"type":"function_call","call_id":1,"name":"read","arguments":"a"}\n' + '{"type":"function_call","call_id":2,"name":"read","arguments":"b"}\n' + '{"type":"function_call_output","call_id":2,"output":"empty"}\n' + '{"type":"function_call_output","call_id":1,"output":"secret"}' ) assert prompt_for("c1", "c2") != prompt_for("c2", "c1") -def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): +def test_redis_semantic_cache_prompt_extraction_replaces_encrypted_reasoning_with_a_digest(): from litellm.caching.redis_semantic_cache import RedisSemanticCache - class ModelDumpInput: - def model_dump(self): - return {"content": [{"text": "model dump prompt"}]} + def prompt_for(encrypted_content: str) -> str | None: + return RedisSemanticCache._get_prompt_from_kwargs( + input=[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": encrypted_content}] + ) - class DictInput: - def dict(self): - return {"content": [{"output_text": "dict prompt"}]} + blob = "gAAAAB" * 100 + prompt = prompt_for(blob) + assert prompt is not None + assert blob not in prompt + assert prompt.startswith('{"type":"reasoning","id":1,"encrypted_content":"sha256:') + assert prompt != prompt_for(blob + "x") + + +def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): + from pydantic import BaseModel + + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + class ResponseInputMessage(BaseModel): + content: list[dict[str, str]] prompt = RedisSemanticCache._get_prompt_from_kwargs( input=[ - ModelDumpInput(), - DictInput(), - {"content": [{"input_text": "inline prompt"}]}, - {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, + ResponseInputMessage(content=[{"type": "input_text", "text": "model dump prompt"}]), + {"content": [{"type": "output_text", "text": "inline prompt", "annotations": []}]}, ] ) - assert prompt == "model dump prompt\ndict prompt\ninline prompt" + assert prompt == "model dump prompt\ninline prompt" -def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): +def test_redis_semantic_cache_prompt_extraction_keeps_image_only_input(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + assert RedisSemanticCache._get_prompt_from_kwargs( + input=[{"type": "input_image", "image_url": "https://example.com/a.png"}] + ) == '{"type":"input_image","image_url":"https://example.com/a.png"}' + + +def test_redis_semantic_cache_prompt_extraction_returns_none_without_input(): from litellm.caching.redis_semantic_cache import RedisSemanticCache assert RedisSemanticCache._get_prompt_from_kwargs() is None assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None - assert ( - RedisSemanticCache._get_prompt_from_kwargs( - input=[{"type": "input_image", "image_url": "https://example.com"}] - ) - is None - ) - - -def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - prompt = RedisSemanticCache._get_prompt_from_kwargs( - input={"text": " ", "input_text": "fallback prompt"} - ) - - assert prompt == "fallback prompt" - - -def test_redis_semantic_cache_prompt_extraction_skips_blank_object_text_keys(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - class ResponseInput: - text = " " - input_text = "fallback prompt" - - prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) - - assert prompt == "fallback prompt" - - -def test_redis_semantic_cache_prompt_extraction_handles_object_content(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - class ResponseInput: - content = [{"text": "object content prompt"}] - - prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) - - assert prompt == "object content prompt" + assert RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_text", "text": " "}]) is None def test_redis_semantic_cache_set_cache_skips_blank_responses_input(): diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 59663de68d0..468bc71f95d 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -3,6 +3,7 @@ import functools import json import os import sys +from collections.abc import Callable from typing import Final import pytest @@ -2023,12 +2024,12 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}], }, ], - 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}', - id="anthropic-tool-use-name-and-input-without-id", + 'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}', + id="anthropic-tool-use-with-its-call-position", ), pytest.param( [_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}], - 'fix the failing test{"result_of_call":null,"output":"calc.py"}', + 'fix the failing test{"type":"tool_result","tool_use_id":1,"content":"calc.py"}', id="anthropic-string-tool-result", ), pytest.param( @@ -2045,7 +2046,7 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} ], } ], - '{"result_of_call":null,"output":"ab"}next', + '{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next', id="anthropic-nested-text-tool-result-then-text", ), pytest.param( @@ -2061,9 +2062,11 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} }, {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ], - 'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}' - '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1,"output":"ok"}', - id="openai-tool-calls-in-order-before-tool-result", + 'fix the failing test{"role":"assistant","content":"writing","tool_calls":[' + '{"id":1,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}},' + '{"id":2,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}}]}' + '{"role":"tool","tool_call_id":1,"content":"ok"}', + id="openai-tool-calls-then-tool-result", ), pytest.param( [ @@ -2082,8 +2085,9 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} ], }, ], - '{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}' - '{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}', + '{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}' + '{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}' + '{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}', id="anthropic-parallel-tool-results-tagged-with-their-call", ), pytest.param( @@ -2093,19 +2097,11 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} "content": None, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}], }, - { - "role": "assistant", - "content": None, - "tool_calls": [ - {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, - {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, - ], - }, - {"role": "tool", "tool_call_id": "call_1", "content": "C"}, + {"role": "tool", "tool_call_id": "call_0", "content": "C"}, ], - '{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}' - '{"result_of_call":2,"output":"C"}', - id="reused-call-ids-keep-first-position", + '{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}' + '{"role":"tool","tool_call_id":1,"content":"C"}', + id="null-content-dropped-from-encoded-message", ), pytest.param( [ @@ -2116,13 +2112,13 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} "search_results": [{"source": "s", "title": "t"}], } ], - '{"result_of_call":null,"output":"small"}st', - id="tool-result-search-results-follow-the-encoded-result", + '{"role":"tool","tool_call_id":1,"content":"small","search_results":[{"source":"s","title":"t"}]}', + id="tool-result-keeps-search-results", ), pytest.param( - [{"role": "tool", "tool_call_id": "c1", "content": '"},{"result_of_call":2,"output":"'}], - '{"result_of_call":null,"output":"\\"},{\\"result_of_call\\":2,\\"output\\":\\""}', - id="tool-output-cannot-forge-an-encoded-result", + [{"role": "tool", "content": 'x"}{"role":"tool","content":"y'}], + '{"role":"tool","content":"x\\"}{\\"role\\":\\"tool\\",\\"content\\":\\"y"}', + id="tool-output-is-escaped-even-without-a-call-id", ), pytest.param( [ @@ -2131,17 +2127,32 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} tool_calls=[ChatCompletionMessageToolCall(id="c1", function=Function(name="read", arguments="{}"))], ) ], - '{"name":"read","arguments":"{}"}', + '{"role":"assistant","tool_calls":[{"function":{"arguments":"{}","name":"read"},"id":1,"type":"function"}]}', id="openai-response-message-object", ), pytest.param( - [{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}, "junk"], - '{"name":null,"arguments":null}', - id="malformed-tool-call-entries", + [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], + "hi", + id="cache-control-ignored", + ), + pytest.param( + [{"role": "user", "name": "alice", "content": "hi"}], + '{"role":"user","name":"alice","content":"hi"}', + id="extra-message-field-kept", + ), + pytest.param( + [{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}], + '{"type":"brand_new_block","payload":{"k":"v"}}!', + id="unknown-block-type-kept", + ), + pytest.param( + [{"role": "user", "content": [{"type": "text", "text": 3}]}], + '{"type":"text","text":3}', + id="non-string-text-kept", ), ], ) -def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[object], expected: str) -> None: +def test_get_str_from_messages_with_tools_keeps_every_part(messages: list[object], expected: str) -> None: assert get_str_from_messages_with_tools(messages) == expected @@ -2150,25 +2161,54 @@ def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[obj [ pytest.param([_TASK, {"role": "assistant", "content": "done"}], id="string-content"), pytest.param( - [ - { - "role": "user", - "content": [ - {"type": "text", "text": "what is "}, - {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, - {"type": "text", "text": "this"}, - ], - } - ], - id="text-and-image-parts", + [{"role": "user", "content": [{"type": "text", "text": "what is "}, {"type": "text", "text": "this"}]}], + id="text-parts", ), pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"), ], ) -def test_get_str_from_messages_with_tools_matches_get_str_from_messages_without_tools(messages: list[object]) -> None: +def test_get_str_from_messages_with_tools_matches_get_str_from_messages_for_text(messages: list[object]) -> None: assert get_str_from_messages_with_tools(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures +def _anthropic_image(data: str) -> list[object]: + return [{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "data": data}}]}] + + +def _openai_image(url: str) -> list[object]: + return [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": url}}]}] + + +def _thinking(signature: str) -> list[object]: + return [{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": signature}]}] + + +@pytest.mark.parametrize( + ("build", "blob"), + [ + pytest.param(_anthropic_image, "iVBORw0KGgo" * 50, id="anthropic-base64-image"), + pytest.param(_openai_image, "data:image/png;base64," + "iVBORw0KGgo" * 50, id="openai-data-url-image"), + pytest.param(_thinking, "EqQBCkgIARABGAIiQL" * 20, id="thinking-signature"), + ], +) +def test_get_str_from_messages_with_tools_replaces_opaque_blobs_with_distinct_digests( + build: Callable[[str], list[object]], blob: str +) -> None: + prompt: Final = get_str_from_messages_with_tools(build(blob)) + + assert blob not in prompt + assert "sha256:" in prompt + assert len(prompt) < len(blob) + assert prompt == get_str_from_messages_with_tools(build(blob)) + assert prompt != get_str_from_messages_with_tools(build(blob + "A")) + + +def test_get_str_from_messages_with_tools_keeps_image_urls_distinct() -> None: + assert get_str_from_messages_with_tools(_openai_image("https://example.com/a.png")) != ( + get_str_from_messages_with_tools(_openai_image("https://example.com/b.png")) + ) + + def _parallel_reads(result_for_a: str, result_for_b: str, *, call_id_prefix: str = "c") -> list[object]: return [ {