From ccca38f7aedbaf1a3e94a88f5d4f6f3b94f94855 Mon Sep 17 00:00:00 2001 From: kerry Date: Wed, 30 Sep 2026 23:39:28 +0000 Subject: [PATCH] Revert "feat(caching): embed every semantic cache prompt field except volatile ones" This reverts commit 5aed3ab3de3597dec22d3afcb33db7196f0a2a24. --- litellm-rust/Cargo.lock | 1 - .../cache-qdrant-semantic/tests/qdrant.rs | 8 +- litellm-rust/crates/cache/Cargo.toml | 1 - litellm-rust/crates/cache/src/semantic.rs | 410 +++++++++--------- litellm-rust/crates/cache/tests/semantic.rs | 407 ++++++++++------- litellm/caching/redis_semantic_cache.py | 100 ++++- .../prompt_templates/common_utils.py | 188 +++----- .../unit/caching/test_redis_semantic_cache.py | 128 ++++-- ...ore_utils_prompt_templates_common_utils.py | 126 ++---- 9 files changed, 751 insertions(+), 618 deletions(-) diff --git a/litellm-rust/Cargo.lock b/litellm-rust/Cargo.lock index 63b4c935208..8d189c8c515 100644 --- a/litellm-rust/Cargo.lock +++ b/litellm-rust/Cargo.lock @@ -3426,7 +3426,6 @@ dependencies = [ "rstest", "serde", "serde_json", - "sha2 0.10.9", "thiserror 2.0.19", "tokio", ] diff --git a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs index 2d94d766146..7a27519c269 100644 --- a/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs +++ b/litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs @@ -225,7 +225,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { } #[rstest] -#[case::content_parts_keep_images_in_place( +#[case::content_parts_skip_images( json!([ {"role": "user", "content": "hello"}, { @@ -237,9 +237,9 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { ], }, ]), - r#"helloworld{"type":"image_url","image_url":{"url":"ignored"}}!"# + "helloworld!" )] -#[case::tool_message_with_search_results_kept_as_json( +#[case::search_results_and_compact_citations( json!([{ "role": "tool", "content": null, @@ -250,7 +250,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) { "citations": {"page": 1, "section": "intro"}, }], }]), - r#"{"role":"tool","search_results":[{"source":"source","title":"title","content":[{"text":"body"}],"citations":{"page":1,"section":"intro"}}]}"# + r#"{"result_of_call":null,"output":""}sourcetitlebody{"page":1,"section":"intro"}"# )] #[tokio::test(flavor = "multi_thread")] async fn prompt_matches_python_message_rules( diff --git a/litellm-rust/crates/cache/Cargo.toml b/litellm-rust/crates/cache/Cargo.toml index 69b898adb11..f18dbd9cb26 100644 --- a/litellm-rust/crates/cache/Cargo.toml +++ b/litellm-rust/crates/cache/Cargo.toml @@ -8,7 +8,6 @@ repository.workspace = true [dependencies] serde.workspace = true serde_json = { workspace = true, features = ["preserve_order"] } -sha2.workspace = true thiserror.workspace = true [dev-dependencies] diff --git a/litellm-rust/crates/cache/src/semantic.rs b/litellm-rust/crates/cache/src/semantic.rs index db14abee664..253efb47759 100644 --- a/litellm-rust/crates/cache/src/semantic.rs +++ b/litellm-rust/crates/cache/src/semantic.rs @@ -8,10 +8,9 @@ use std::{collections::HashMap, future::Future, io}; use serde::Serialize; use serde_json::{ - Map, Value, + Value, json, ser::{CharEscape, Formatter, Serializer}, }; -use sha2::{Digest, Sha256}; use crate::{BaseCache, Error, SemanticCacheContext}; @@ -84,227 +83,124 @@ impl Embedder for PreparedEmbedding { } } -const PLAIN_TYPES: [&str; 4] = ["text", "input_text", "output_text", "message"]; -const PLAIN_KEYS: [&str; 5] = ["role", "type", "text", "content", "status"]; -const TOOL_ROLES: [&str; 2] = ["tool", "function"]; -const IGNORED_KEYS: [&str; 1] = ["cache_control"]; -const CALL_ID_KEYS: [&str; 4] = ["id", "call_id", "tool_use_id", "tool_call_id"]; -const OPAQUE_KEYS: [&str; 4] = ["data", "file_data", "signature", "encrypted_content"]; - -/// `get_str_from_messages_with_tools`: plain text parts stay plain text, and every other part is -/// kept as compact JSON with call ids replaced by their position and opaque blobs by a digest. +/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results, +/// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the +/// position of the call it answers. pub fn str_from_messages(messages: &[Value]) -> String { - let messages: Vec<&Value> = messages - .iter() - .filter(|message| message.is_object()) - .collect(); - let call_ordinals = call_id_ordinals(messages.iter().copied()); - messages - .into_iter() - .filter_map(Value::as_object) - .map(|message| message_prompt(message, &call_ordinals)) - .collect() -} - -/// `get_str_from_responses_input`: `str_from_messages` for a Responses API `input`, one stripped -/// line per part. `None` when nothing is left. -fn str_from_responses_input(input: &Value) -> Option { - let call_ordinals = call_id_ordinals(std::iter::once(input)); - let mut parts = Vec::new(); - push_responses_input_parts(input, &call_ordinals, &mut parts); - let prompt = python_strip(&parts.join("\n")).to_owned(); - (!prompt.is_empty()).then_some(prompt) -} - -fn message_prompt(message: &Map, call_ordinals: &HashMap<&str, usize>) -> String { - let is_tool = message - .get("role") - .and_then(Value::as_str) - .is_some_and(|role| TOOL_ROLES.contains(&role)); - if is_tool || !is_plain(message) { - return compact_json(&normalized(message, call_ordinals)); + let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect(); + let call_ordinals = tool_call_ordinals(messages.iter().flat_map(|message| { + let block_ids = message + .get("content") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_use")) + .filter_map(|block| block.get("id")); + let tool_call_ids = message + .get("tool_calls") + .and_then(Value::as_array) + .into_iter() + .flatten() + .filter_map(|tool_call| tool_call.get("id")); + block_ids.chain(tool_call_ids) + })); + let mut text = String::new(); + for message in messages { + if message.get("role").and_then(Value::as_str) == Some("tool") { + let mut output = String::new(); + push_content_text(&mut output, message.get("content"), &call_ordinals); + text.push_str(&tool_result_json( + message.get("tool_call_id"), + &call_ordinals, + &output, + )); + } else { + push_content_text(&mut text, message.get("content"), &call_ordinals); + } + if let Some(Value::Array(tool_calls)) = message.get("tool_calls") { + for tool_call in tool_calls.iter().filter_map(Value::as_object) { + let function = tool_call.get("function"); + text.push_str(&tool_call_json( + function.and_then(|function| function.get("name")), + function.and_then(|function| function.get("arguments")), + )); + } + } + push_search_results_text(&mut text, message.get("search_results")); } - plain_prompt(message, call_ordinals) + text } -fn plain_prompt(value: &Map, call_ordinals: &HashMap<&str, usize>) -> String { - let mut prompt = value - .get("text") - .and_then(Value::as_str) - .unwrap_or_default() - .to_owned(); - push_content_prompt(&mut prompt, value.get("content"), call_ordinals); - prompt -} - -fn push_content_prompt( - prompt: &mut String, +/// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content. +fn push_content_text( + text: &mut String, content: Option<&Value>, call_ordinals: &HashMap<&str, usize>, ) { match content { - None | Some(Value::Null) => {} - Some(Value::String(text)) => prompt.push_str(text), - Some(Value::Array(items)) => { - for item in items { - push_content_prompt(prompt, Some(item), call_ordinals); - } - } - Some(Value::Object(map)) if is_plain(map) => { - prompt.push_str(&plain_prompt(map, call_ordinals)); - } - Some(Value::Object(map)) => prompt.push_str(&compact_json(&normalized(map, call_ordinals))), - Some(other) => prompt.push_str(&compact_json(other)), - } -} - -fn push_responses_input_parts( - value: &Value, - call_ordinals: &HashMap<&str, usize>, - parts: &mut Vec, -) { - match value { - Value::Null => {} - Value::String(text) => push_stripped(text, parts), - Value::Array(items) => { - for item in items { - push_responses_input_parts(item, call_ordinals, parts); - } - } - Value::Object(map) if is_plain(map) => { - push_stripped( - map.get("text").and_then(Value::as_str).unwrap_or_default(), - parts, - ); - if let Some(content) = map.get("content") { - push_responses_input_parts(content, call_ordinals, parts); - } - } - Value::Object(map) => parts.push(compact_json(&normalized(map, call_ordinals))), - other => parts.push(compact_json(other)), - } -} - -fn push_stripped(text: &str, parts: &mut Vec) { - let stripped = python_strip(text); - if !stripped.is_empty() { - parts.push(stripped.to_owned()); - } -} - -/// `_is_plain`: a text-only part whose every kept key is a plain or call id key. -fn is_plain(value: &Map) -> bool { - let plain_type = match value.get("type") { - None | Some(Value::Null) => true, - Some(Value::String(value_type)) => PLAIN_TYPES.contains(&value_type.as_str()), - Some(_) => false, - }; - let plain_text = matches!( - value.get("text"), - None | Some(Value::Null | Value::String(_)) - ); - plain_type - && plain_text - && kept_entries(value).all(|(key, _)| { - PLAIN_KEYS.contains(&key.as_str()) || CALL_ID_KEYS.contains(&key.as_str()) - }) -} - -fn kept_entries(value: &Map) -> impl Iterator { - value - .iter() - .filter(|(key, item)| !IGNORED_KEYS.contains(&key.as_str()) && !is_empty(item)) -} - -fn is_empty(value: &Value) -> bool { - match value { - Value::Null => true, - Value::String(text) => text.is_empty(), - Value::Array(items) => items.is_empty(), - Value::Object(map) => map.is_empty(), - Value::Bool(_) | Value::Number(_) => false, - } -} - -/// `_normalized`: the part minus ignored and empty entries, with call ids replaced by their -/// position and opaque blobs by `sha256:` plus the first 16 hex digits of their digest. -fn normalized(value: &Map, call_ordinals: &HashMap<&str, usize>) -> Value { - Value::Object( - kept_entries(value) - .map(|(key, item)| { - ( - key.clone(), - normalized_value(item, call_ordinals, Some(key)), - ) - }) - .collect(), - ) -} - -fn normalized_value( - value: &Value, - call_ordinals: &HashMap<&str, usize>, - key: Option<&str>, -) -> Value { - match value { - Value::Object(map) => normalized(map, call_ordinals), - Value::Array(items) => Value::Array( - items - .iter() - .map(|item| normalized_value(item, call_ordinals, None)) - .collect(), - ), - Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => call_ordinals - .get(text.as_str()) - .map_or_else(|| value.clone(), |ordinal| Value::from(*ordinal)), - Value::String(text) - if key.is_some_and(|key| OPAQUE_KEYS.contains(&key)) || text.starts_with("data:") => - { - Value::String(digest(text)) - } - other => other.clone(), - } -} - -fn digest(text: &str) -> String { - let hash = Sha256::digest(text.as_bytes()); - let hex: String = hash[..8].iter().map(|byte| format!("{byte:02x}")).collect(); - format!("sha256:{hex}") -} - -/// `_call_id_ordinals`: the 1-based position of each distinct call id string, first seen first. -fn call_id_ordinals<'a>(values: impl Iterator) -> HashMap<&'a str, usize> { - let mut ids = Vec::new(); - for value in values { - collect_call_ids(value, None, &mut ids); - } - let mut ordinals = HashMap::new(); - for id in ids { - let next = ordinals.len() + 1; - ordinals.entry(id).or_insert(next); - } - ordinals -} - -fn collect_call_ids<'a>(value: &'a Value, key: Option<&str>, ids: &mut Vec<&'a str>) { - match value { - Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => ids.push(text), - Value::Object(map) => { - for (child_key, child) in map { - if !IGNORED_KEYS.contains(&child_key.as_str()) { - collect_call_ids(child, Some(child_key), ids); + Some(Value::String(content)) => text.push_str(content), + Some(Value::Array(blocks)) => { + for block in blocks.iter().filter_map(Value::as_object) { + match block.get("type").and_then(Value::as_str) { + Some("tool_use") => { + text.push_str(&tool_call_json(block.get("name"), block.get("input"))); + } + Some("tool_result") => { + let mut output = String::new(); + push_content_text(&mut output, block.get("content"), call_ordinals); + text.push_str(&tool_result_json( + block.get("tool_use_id"), + call_ordinals, + &output, + )); + } + _ => { + if let Some(block_text) = block.get("text").and_then(Value::as_str) { + text.push_str(block_text); + } + } } } } - Value::Array(items) => { - for item in items { - collect_call_ids(item, None, ids); - } - } _ => {} } } +/// `tool_call_ordinals`: the 1-based position of each distinct string call id, first seen first. +fn tool_call_ordinals<'a>(call_ids: impl Iterator) -> HashMap<&'a str, usize> { + let mut ordinals = HashMap::new(); + for call_id in call_ids.filter_map(Value::as_str) { + let next = ordinals.len() + 1; + ordinals.entry(call_id).or_insert(next); + } + ordinals +} + +/// `tool_result_str`: `{"result_of_call":N,"output":...}`, with a `null` position when the result +/// answers no known call. +fn tool_result_json( + call_id: Option<&Value>, + call_ordinals: &HashMap<&str, usize>, + output: &str, +) -> String { + let ordinal = call_id + .and_then(Value::as_str) + .and_then(|call_id| call_ordinals.get(call_id)); + format!( + "{{\"result_of_call\":{},\"output\":{}}}", + compact_json(&json!(ordinal)), + compact_json(&Value::String(output.to_owned())), + ) +} + +/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes. +fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String { + compact_json(&json!({ + "name": name.unwrap_or(&Value::Null), + "arguments": arguments.unwrap_or(&Value::Null), + })) +} + /// The messages prompt Qdrant embeds: `None` when the request carries no messages. pub fn prompt_from_messages(context: &SemanticCacheContext) -> Option { let messages = context.messages.as_ref()?.as_array()?; @@ -319,7 +215,103 @@ pub fn prompt_from_context(context: &SemanticCacheContext) -> Option { { return Some(str_from_messages(messages)); } - str_from_responses_input(context.input.as_ref()?) + let input = context.input.as_ref()?; + let call_ordinals = tool_call_ordinals( + input + .as_array() + .into_iter() + .flatten() + .filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call")) + .filter_map(|item| item.get("call_id")), + ); + let mut parts = Vec::new(); + collect_input_text(input, &mut parts, &call_ordinals); + let prompt = python_strip(&parts.join("\n")).to_owned(); + (!prompt.is_empty()).then_some(prompt) +} + +/// `extract_search_results_text`. +fn push_search_results_text(text: &mut String, search_results: Option<&Value>) { + let Some(Value::Array(results)) = search_results else { + return; + }; + for result in results.iter().filter_map(Value::as_object) { + for key in ["source", "title"] { + if let Some(value) = result.get(key).and_then(Value::as_str) { + text.push_str(value); + } + } + if let Some(Value::Array(content)) = result.get("content") { + for block in content.iter().filter_map(Value::as_object) { + if let Some(value) = block.get("text").and_then(Value::as_str) { + text.push_str(value); + } + } + } + if let Some(citations) = result.get("citations").filter(|value| !value.is_null()) { + text.push_str(&compact_json(citations)); + } + } +} + +fn collect_input_text( + value: &Value, + parts: &mut Vec, + call_ordinals: &HashMap<&str, usize>, +) { + match value { + Value::String(text) => { + push_trimmed(text, parts); + } + Value::Array(items) => { + for item in items { + collect_input_text(item, parts, call_ordinals); + } + } + Value::Object(map) => { + if map.get("type").and_then(Value::as_str) == Some("function_call") { + parts.push(tool_call_json(map.get("name"), map.get("arguments"))); + return; + } + if map.get("type").and_then(Value::as_str) == Some("function_call_output") { + let mut output_parts = Vec::new(); + if let Some(output) = map.get("output") { + collect_input_text(output, &mut output_parts, call_ordinals); + } + parts.push(tool_result_json( + map.get("call_id"), + call_ordinals, + python_strip(&output_parts.join("\n")), + )); + return; + } + if let Some(content) = map.get("content").filter(|content| !content.is_null()) { + collect_input_text(content, parts, call_ordinals); + return; + } + for key in ["text", "output", "input_text", "output_text"] { + match map.get(key) { + Some(nested @ Value::Array(_)) => { + collect_input_text(nested, parts, call_ordinals); + return; + } + Some(Value::String(text)) if push_trimmed(text, parts) => return, + _ => {} + } + } + } + _ => {} + } +} + +/// Pushes `text` stripped as Python's `str.strip` does, reporting whether anything was left. +fn push_trimmed(text: &str, parts: &mut Vec) -> bool { + let trimmed = python_strip(text); + if trimmed.is_empty() { + return false; + } + parts.push(trimmed.to_owned()); + true } /// `str.strip()`: Python's whitespace also covers the ASCII information separators. diff --git a/litellm-rust/crates/cache/tests/semantic.rs b/litellm-rust/crates/cache/tests/semantic.rs index c4a1bfb2d56..af87e6e0164 100644 --- a/litellm-rust/crates/cache/tests/semantic.rs +++ b/litellm-rust/crates/cache/tests/semantic.rs @@ -16,115 +16,168 @@ fn context(messages: Option, input: Option) -> SemanticCacheContex } #[rstest] -#[case::empty(r#"[]"#, r#""#)] -#[case::string_content(r#"[{"role": "user", "content": "hello"}]"#, r#"hello"#)] +#[case::empty(json!([]), "")] +#[case::string_content(json!([{"role": "user", "content": "hello"}]), "hello")] #[case::concatenates_messages( - r#"[{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]"#, - r#"be brief. hello"# + json!([{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]), + "be brief. hello", )] #[case::text_parts( - r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "text", "text": "this?"}]}]"#, - r#"What is this?"#, -)] -#[case::image_part_kept_in_place( - r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "this?"}]}]"#, - r#"What is {"type":"image_url","image_url":{"url":"https://example.com/a.png"}}this?"#, + json!([{"role": "user", "content": [ + {"type": "text", "text": "What is "}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "text", "text": "this?"}, + ]}]), + "What is this?", )] #[case::missing_null_and_empty_content( - r#"[{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]"#, - r#""#, + json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]), + "", )] -#[case::non_object_messages_skipped( - r#"["junk", 3, {"role": "user", "content": "kept"}]"#, - r#"kept"# +#[case::search_results_hidden_behind_small_content( + json!([{"role": "tool", "content": "small", "search_results": [ + {"source": "s", "title": "t", "content": [{"text": "hidden payload"}]}, + ]}]), + r#"{"result_of_call":null,"output":"small"}sthidden payload"#, )] -#[case::cache_control_ignored( - r#"[{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}]"#, - r#"hi"#, +#[case::title_only_search_result( + json!([{"role": "tool", "content": "small", "search_results": [ + {"source": "s", "title": "long title", "content": []}, + ]}]), + r#"{"result_of_call":null,"output":"small"}slong title"#, )] -#[case::extra_message_field_kept( - r#"[{"role": "user", "name": "alice", "content": "hi"}]"#, - r#"{"role":"user","name":"alice","content":"hi"}"# +#[case::search_results_without_content( + json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]), + r#"{"result_of_call":null,"output":""}st"#, )] -#[case::unknown_block_type_kept( - r#"[{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}]"#, - r#"{"type":"brand_new_block","payload":{"k":"v"}}!"#, +#[case::search_result_fields_in_python_order( + json!([{"role": "tool", "content": "c", "search_results": [ + {"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"}, + {"source": "s2"}, + ]}]), + r#"{"result_of_call":null,"output":"c"}stbody{"enabled":true}s2"#, )] -#[case::non_string_text_kept( - r#"[{"role": "user", "content": [{"type": "text", "text": 3}]}]"#, - r#"{"type":"text","text":3}"# +#[case::null_citations_skipped( + json!([{"role": "tool", "content": "c", "search_results": [ + {"source": "s", "citations": null}, + ]}]), + r#"{"result_of_call":null,"output":"c"}s"#, )] -#[case::scalar_content_kept( - r#"[{"role": "user", "content": 7}, {"role": "user", "content": [1.5, true, "x"]}]"#, - r#"71.5truex"# +#[case::non_string_and_non_object_entries_skipped( + json!([{"role": "tool", "content": "c", "search_results": [ + "junk", + {"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]}, + ]}]), + r#"{"result_of_call":null,"output":"c"}kept"#, )] -#[case::search_results_kept( - r#"[{"role": "tool", "content": "small", "search_results": [{"source": "s", "title": "t", "content": [{"type": "text", "text": "hidden payload"}], "citations": {"enabled": true}}]}]"#, - r#"{"role":"tool","content":"small","search_results":[{"source":"s","title":"t","content":[{"type":"text","text":"hidden payload"}],"citations":{"enabled":true}}]}"#, +#[case::non_list_search_results_skipped( + json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]), + r#"{"result_of_call":null,"output":"c"}"#, )] -#[case::anthropic_tool_use( - r#"[{"role": "user", "content": "fix the failing test"}, {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}]}]"#, - r#"fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}"#, +#[case::citations_compact_in_insertion_order( + json!([{"role": "tool", "search_results": [ + {"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}}, + ]}]), + r#"{"result_of_call":null,"output":""}{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#, +)] +#[case::citations_ensure_ascii( + json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]), + r#"{"result_of_call":null,"output":""}["caf\u00e9","\u4e2d"]"#, +)] +#[case::citations_astral_chars_as_surrogate_pairs( + json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]), + r#"{"result_of_call":null,"output":""}"\ud83d\ude00""#, +)] +#[case::citations_escapes( + json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]), + r#"{"result_of_call":null,"output":""}"q\"\\\n\t\u0001/""#, +)] +#[case::citations_large_float_exponent( + json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]), + r#"{"result_of_call":null,"output":""}[1e+20,1.0]"#, +)] +#[case::citations_scalars( + json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]), + r#"{"result_of_call":null,"output":""}false3"#, +)] +#[case::anthropic_tool_use_name_and_input_without_id( + json!([ + {"role": "user", "content": "fix the failing test"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}, + ]}, + ]), + r#"fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}"#, )] #[case::anthropic_string_tool_result( - r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}]"#, - r#"{"type":"tool_result","tool_use_id":1,"content":"calc.py"}"#, + json!([{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}, + ]}]), + r#"{"result_of_call":null,"output":"calc.py"}"#, )] #[case::anthropic_nested_text_tool_result_then_text( - r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}, {"type": "text", "text": "next"}]}]"#, - r#"{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next"#, + json!([{"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": [ + {"type": "text", "text": "a"}, + {"type": "text", "text": "b"}, + ]}, + {"type": "text", "text": "next"}, + ]}]), + r#"{"result_of_call":null,"output":"ab"}next"#, )] -#[case::openai_tool_calls_then_tool_result( - r#"[{"role": "assistant", "content": "writing", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}]}, {"role": "tool", "tool_call_id": "c1", "content": "ok"}]"#, - r#"{"role":"assistant","content":"writing","tool_calls":[{"id":1,"type":"function","function":{"name":"write","arguments":"{\"path\": \"a\"}"}},{"id":2,"type":"function","function":{"name":"write","arguments":"{\"path\": \"b\"}"}}]}{"role":"tool","tool_call_id":1,"content":"ok"}"#, +#[case::openai_tool_calls_in_order_before_tool_result( + json!([ + {"role": "assistant", "content": "writing", "tool_calls": [ + {"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, + {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}, + ]}, + {"role": "tool", "tool_call_id": "c1", "content": "ok"}, + ]), + r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1,"output":"ok"}"#, )] #[case::anthropic_parallel_tool_results_tagged_with_their_call( - r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t2", "content": "B"}, {"type": "tool_result", "tool_use_id": "t1", "content": "A"}]}]"#, - r#"{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}"#, + json!([ + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, + {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t2", "content": "B"}, + {"type": "tool_result", "tool_use_id": "t1", "content": "A"}, + ]}, + ]), + r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}"#, )] #[case::reused_call_ids_keep_first_position( - r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}]}, {"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}]}, {"role": "tool", "tool_call_id": "call_1", "content": "C"}]"#, - r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"b","arguments":"{}"}},{"id":2,"type":"function","function":{"name":"c","arguments":"{}"}}]}{"role":"tool","tool_call_id":2,"content":"C"}"#, + json!([ + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}, + ]}, + {"role": "assistant", "content": null, "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, + {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, + ]}, + {"role": "tool", "tool_call_id": "call_1", "content": "C"}, + ]), + r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2,"output":"C"}"#, )] -#[case::tool_output_cannot_forge_a_record( - r#"[{"role": "tool", "content": "x\"}{\"role\":\"tool\",\"content\":\"y"}]"#, - r#"{"role":"tool","content":"x\"}{\"role\":\"tool\",\"content\":\"y"}"# +#[case::unknown_call_ids_encode_a_null_position( + json!([ + {"role": "tool", "tool_call_id": "c9", "content": "ok"}, + {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]}, + ]), + r#"{"result_of_call":null,"output":"ok"}{"result_of_call":null,"output":"done"}"#, +)] +#[case::tool_output_cannot_forge_an_encoded_result( + json!([{"role": "tool", "tool_call_id": "c1", "content": "\"},{\"result_of_call\":2,\"output\":\""}]), + r#"{"result_of_call":null,"output":"\"},{\"result_of_call\":2,\"output\":\""}"#, )] #[case::malformed_tool_call_entries( - r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]"#, - r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function"},"junk"]}"#, + json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]), + r#"{"name":null,"arguments":null}"#, )] -#[case::anthropic_base64_image_digested( - r#"[{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#, - r#"{"type":"image","source":{"type":"base64","media_type":"image/png","data":"sha256:b1a66f8de3276946"}}"#, -)] -#[case::openai_data_url_digested( - r#"[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#, - r#"{"type":"image_url","image_url":{"url":"sha256:19c7f028a60a33b8"}}"#, -)] -#[case::thinking_signature_digested( - r#"[{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": "EqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARAB"}, {"type": "text", "text": "done"}]}]"#, - r#"{"type":"thinking","thinking":"hmm","signature":"sha256:b20cbbd82e4b3791"}done"#, -)] -#[case::non_ascii_escaped( - r#"[{"role": "tool", "content": "caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}]"#, - r#"{"role":"tool","content":"caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}"# -)] -#[case::non_ascii_plain_text_kept_raw( - r#"[{"role": "user", "content": "caf\u00e9 \ud83d\ude00"}]"#, - r#"café 😀"# -)] -#[case::floats_in_python_repr( - r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {"a": [1e+20, 1.0, 1.5e-07, 0.0001, 1e+16, -2.5, 3]}}]}]"#, - r#"{"type":"tool_use","id":1,"name":"n","input":{"a":[1e+20,1.0,1.5e-07,0.0001,1e+16,-2.5,3]}}"#, -)] -#[case::empty_fields_dropped( - r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {}, "extra": [], "note": ""}]}]"#, - r#"{"type":"tool_use","id":1,"name":"n"}"#, -)] -fn str_from_messages_matches_python(#[case] messages: &str, #[case] expected: &str) { - let messages: Vec = serde_json::from_str(messages).unwrap(); - assert_eq!(str_from_messages(&messages), expected); +fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) { + assert_eq!(str_from_messages(messages.as_array().unwrap()), expected); } #[rstest] @@ -143,109 +196,167 @@ fn prompt_from_messages_reads_messages_only( #[rstest] #[case::prefers_messages( - Some(r#"[{"content": "message prompt"}]"#), - Some(r#""responses prompt""#), - Some(r#"message prompt"#) + Some(json!([{"content": "message prompt"}])), + Some(json!("responses prompt")), + Some("message prompt"), )] #[case::empty_messages_fall_back_to_input( - Some(r#"[]"#), - Some(r#""responses prompt""#), - Some(r#"responses prompt"#) + Some(json!([])), + Some(json!("responses prompt")), + Some("responses prompt"), )] #[case::messages_without_text_keep_an_empty_prompt( - Some(r#"[{"content": null}]"#), - Some(r#""x""#), - Some(r#""#) + Some(json!([{"content": null}])), + Some(json!("x")), + Some(""), )] -#[case::null_input(None, None, None)] -#[case::blank_string(None, Some(r#"" ""#), None)] +#[case::nothing(None, None, None)] +#[case::null_input(None, Some(Value::Null), None)] +#[case::blank_string(None, Some(json!(" ")), None)] #[case::trimmed_string( None, - Some(r#"" What is the capital of France?\n""#), - Some(r#"What is the capital of France?"#) + Some(json!(" What is the capital of France?\n")), + Some("What is the capital of France?"), )] -#[case::strip_information_separators(None, Some(r#""\u001ca\u001f""#), Some(r#"a"#))] -#[case::image_only_kept( +#[case::image_only( + None, + Some(json!([{"type": "input_image", "image_url": "https://example.com"}])), None, - Some(r#"[{"type": "input_image", "image_url": "https://example.com"}]"#), - Some(r#"{"type":"input_image","image_url":"https://example.com"}"#) )] #[case::structured_input( None, - Some(r#"[{"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}, {"type": "input_text", "text": "Answer briefly."}, {"type": "input_image", "image_url": "https://example.com/paris.png"}]}]"#), - Some(r#"What is the capital of France? -Answer briefly. -{"type":"input_image","image_url":"https://example.com/paris.png"}"#), + Some(json!([{"role": "user", "content": [ + {"type": "input_text", "text": "What is the capital of France?"}, + {"type": "input_text", "text": "Answer briefly."}, + {"type": "input_image", "image_url": "https://example.com/paris.png"}, + ]}])), + Some("What is the capital of France?\nAnswer briefly."), )] -#[case::message_items_with_ids( +#[case::model_objects_after_dump( None, - Some(r#"[{"type": "message", "id": "msg_1", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "hi", "annotations": []}]}]"#), - Some(r#"hi"#), + Some(json!([ + {"content": [{"text": "model dump prompt"}]}, + {"content": [{"output_text": "dict prompt"}]}, + {"content": [{"input_text": "inline prompt"}]}, + {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, + ])), + Some("model dump prompt\ndict prompt\ninline prompt"), )] #[case::object_content( None, - Some(r#"{"content": [{"text": "object content prompt"}]}"#), - Some(r#"object content prompt"#) + Some(json!({"content": [{"text": "object content prompt"}]})), + Some("object content prompt"), )] -#[case::string_content(None, Some(r#"{"content": " inline "}"#), Some(r#"inline"#))] -#[case::unknown_keys_kept( +#[case::string_content(None, Some(json!({"content": " inline "})), Some("inline"))] +#[case::null_content_uses_text_keys( None, - Some(r#"{"text": " ", "input_text": "fallback prompt"}"#), - Some(r#"{"text":" ","input_text":"fallback prompt"}"#) + Some(json!({"content": null, "output": "tool output"})), + Some("tool output"), )] -#[case::nested_lists( +#[case::content_wins_over_text(None, Some(json!({"content": [], "text": "ignored"})), None)] +#[case::text_key_precedence( None, - Some(r#"[["a", [" b "]], "", "c"]"#), - Some( - r#"a -b -c"# - ) + Some(json!({"output_text": "d", "input_text": "c", "output": "b", "text": "a"})), + Some("a"), )] -#[case::scalars_kept( +#[case::input_text_key(None, Some(json!({"input_text": "only input"})), Some("only input"))] +#[case::output_text_key(None, Some(json!({"output_text": "only output"})), Some("only output"))] +#[case::non_string_text_keys_skipped( None, - Some(r#"[1, true, null, "kept"]"#), - Some( - r#"1 -true -kept"# - ) + Some(json!({"text": 1, "output": "fallback"})), + Some("fallback"), )] -#[case::blank_text_parts(None, Some(r#"[{"type": "input_text", "text": " "}]"#), None)] +#[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))] +#[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))] #[case::responses_function_call( None, - Some(r#"[{"role": "user", "content": "update the config"}, {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, {"type": "function_call_output", "call_id": "c1", "output": "ok"}]"#), - Some(r#"update the config -{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\"path\":\"a.yaml\"}"} -{"type":"function_call_output","call_id":1,"output":"ok"}"#), + Some(json!([ + {"role": "user", "content": "update the config"}, + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, + {"type": "function_call_output", "call_id": "c1", "output": "ok"}, + ])), + Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1,\"output\":\"ok\"}"), )] #[case::responses_structured_function_call_output( None, - Some(r#"[{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": " denied "}]}]"#), - Some(r#"{"type":"function_call","call_id":1,"name":"write_file","arguments":"{}"} -{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":" denied "}]}"#), + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]}, + ])), + Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1,\"output\":\"denied\"}"), +)] +#[case::responses_multi_part_output_joined_by_lines( + None, + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, + {"type": "function_call_output", "call_id": "c1", "output": [ + {"type": "input_text", "text": " line one "}, + {"type": "input_text", "text": "line two"}, + ]}, + ])), + Some(r#"{"name":"run","arguments":"{}"} +{"result_of_call":1,"output":"line one\nline two"}"#), )] #[case::responses_parallel_outputs_tagged_with_their_call( None, - Some(r#"[{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, {"type": "function_call_output", "call_id": "c2", "output": "B"}, {"type": "function_call_output", "call_id": "c1", "output": "A"}]"#), - Some(r#"{"type":"function_call","call_id":1,"name":"read","arguments":"a"} -{"type":"function_call","call_id":2,"name":"read","arguments":"b"} -{"type":"function_call_output","call_id":2,"output":"B"} -{"type":"function_call_output","call_id":1,"output":"A"}"#), + Some(json!([ + {"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, + {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, + {"type": "function_call_output", "call_id": "c2", "output": "B"}, + {"type": "function_call_output", "call_id": "c1", "output": "A"}, + ])), + Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2,\"output\":\"B\"}\n{\"result_of_call\":1,\"output\":\"A\"}"), )] -#[case::responses_reasoning_digested( +#[case::responses_unknown_call_id_encodes_a_null_position( None, - Some(r#"[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "gAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAAB"}]"#), - Some(r#"{"type":"reasoning","id":1,"encrypted_content":"sha256:010b9eb723d8d1e2"}"#), + Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])), + Some(r#"{"result_of_call":null,"output":"orphan"}"#), )] fn prompt_from_context_matches_python( - #[case] messages: Option<&str>, - #[case] input: Option<&str>, + #[case] messages: Option, + #[case] input: Option, #[case] expected: Option<&str>, ) { - let parse = |value: &str| serde_json::from_str::(value).unwrap(); - let context = context(messages.map(parse), input.map(parse)); - assert_eq!(prompt_from_context(&context).as_deref(), expected); + assert_eq!( + prompt_from_context(&context(messages, input)).as_deref(), + expected + ); +} + +/// Python `test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys`: a blank +/// text key falls through to the next one. +#[rstest] +#[case::blank_text_falls_through( + json!({"text": " ", "input_text": "fallback prompt"}), + "fallback prompt", +)] +fn prompt_from_context_skips_blank_text_keys(#[case] input: Value, #[case] expected: &str) { + assert_eq!( + prompt_from_context(&context(None, Some(input))).as_deref(), + Some(expected) + ); +} + +/// Where `json.dumps(..., separators=(",", ":"))` and Python `str.strip` differ from +/// `semantic.rs`: ensure_ascii escapes DEL, small floats keep Python's two-digit exponent, and +/// strip also removes the ASCII information separators. +#[rstest] +#[case::del_is_escaped(json!([{"search_results": [{"citations": "\u{7f}"}]}]), None, r#""\u007f""#)] +#[case::small_float_exponent(json!([{"search_results": [{"citations": 1.5e-7}]}]), None, "1.5e-07")] +#[case::float_at_positional_floor(json!([{"search_results": [{"citations": 1e-4}]}]), None, "0.0001")] +#[case::float_at_scientific_ceiling(json!([{"search_results": [{"citations": 1e16}]}]), None, "1e+16")] +#[case::large_float(json!([{"search_results": [{"citations": [1.25e20, -2.5, 3.0]}]}]), None, "[1.25e+20,-2.5,3.0]")] +#[case::strip_information_separators(json!([]), Some(json!("\u{1c}a\u{1f}")), "a")] +fn python_serialization_edge_cases( + #[case] messages: Value, + #[case] input: Option, + #[case] expected: &str, +) { + let actual = match input { + Some(input) => prompt_from_context(&context(None, Some(input))).unwrap_or_default(), + None => str_from_messages(messages.as_array().unwrap()), + }; + assert_eq!(actual, expected); } #[rstest] diff --git a/litellm/caching/redis_semantic_cache.py b/litellm/caching/redis_semantic_cache.py index 9bf88989958..827fa91398f 100644 --- a/litellm/caching/redis_semantic_cache.py +++ b/litellm/caching/redis_semantic_cache.py @@ -22,7 +22,9 @@ from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS from litellm.litellm_core_utils.asyncify import asyncify from litellm.litellm_core_utils.prompt_templates.common_utils import ( get_str_from_messages_with_tools, - get_str_from_responses_input, + tool_call_ordinals, + tool_call_str, + tool_result_str, ) from litellm.types.utils import EmbeddingResponse @@ -269,7 +271,101 @@ class RedisSemanticCache(BaseCache): if "input" not in kwargs: return None - return get_str_from_responses_input(kwargs.get("input")) + responses_input: Final = kwargs.get("input") + prompt: Final = cls._responses_input_prompt(responses_input, cls._responses_call_ordinals(responses_input)) + return prompt or None + + @classmethod + def _responses_input_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> str: + prompt_parts: Final[list[str]] = [] + cls._collect_responses_input_text(value, prompt_parts, call_ordinals) + return "\n".join(prompt_parts).strip() + + @classmethod + def _collect_responses_input_text( + cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int] + ) -> None: + value = cls._function_call_as_prompt(cls._coerce_response_input_value(value), call_ordinals) + if value is None: + return + + if isinstance(value, str): + stripped_value: Final = value.strip() + if stripped_value: + prompt_parts.append(stripped_value) + return + + if isinstance(value, (list, tuple)): + for item in value: + cls._collect_responses_input_text(item, prompt_parts, call_ordinals) + return + + if isinstance(value, dict): + content = value.get("content") + if content is not None: + cls._collect_responses_input_text(content, prompt_parts, call_ordinals) + return + + cls._collect_responses_text_fields(value, prompt_parts, call_ordinals) + return + + content = getattr(value, "content", None) + if content is not None: + cls._collect_responses_input_text(content, prompt_parts, call_ordinals) + return + + for text_key in ("text", "output", "input_text", "output_text"): + text_value = getattr(value, text_key, None) + if isinstance(text_value, str): + stripped_text = text_value.strip() + if stripped_text: + prompt_parts.append(stripped_text) + return + + @classmethod + def _collect_responses_text_fields( + cls, value: dict, prompt_parts: list[str], call_ordinals: Mapping[str, int] + ) -> None: + for text_key in ("text", "output", "input_text", "output_text"): + text_value = value.get(text_key) + if isinstance(text_value, (list, tuple)): + cls._collect_responses_input_text(text_value, prompt_parts, call_ordinals) + return + if isinstance(text_value, str) and (stripped_text := text_value.strip()): + prompt_parts.append(stripped_text) + return + + @classmethod + def _responses_call_ordinals(cls, responses_input: object) -> Mapping[str, int]: + items: Final = responses_input if isinstance(responses_input, (list, tuple)) else () + dumped_items: Final = (cls._coerce_response_input_value(item) for item in items) + return tool_call_ordinals( + item.get("call_id") + for item in dumped_items + if isinstance(item, dict) and item.get("type") == "function_call" + ) + + @classmethod + def _function_call_as_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> object: + if not isinstance(value, dict): + return value + if value.get("type") == "function_call": + return tool_call_str(value.get("name"), value.get("arguments")) + if value.get("type") != "function_call_output": + return value + return tool_result_str( + value.get("call_id"), call_ordinals, cls._responses_input_prompt(value.get("output"), call_ordinals) + ) + + @staticmethod + def _coerce_response_input_value(value: object) -> object: + model_dump: Final = getattr(value, "model_dump", None) + if callable(model_dump): + return model_dump() + dict_method: Final = getattr(value, "dict", None) + if callable(dict_method): + return dict_method() + return value def _embedding_input(self, prompt: str, router: "Router | None") -> str: return truncate_embedding_input( diff --git a/litellm/litellm_core_utils/prompt_templates/common_utils.py b/litellm/litellm_core_utils/prompt_templates/common_utils.py index 8e75ac4034e..0c52fae86d7 100644 --- a/litellm/litellm_core_utils/prompt_templates/common_utils.py +++ b/litellm/litellm_core_utils/prompt_templates/common_utils.py @@ -2,7 +2,6 @@ Common utility functions used for translating messages across providers """ -import hashlib import io import json import mimetypes @@ -196,148 +195,93 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str: return text -_PLAIN_TYPES: Final = frozenset({"text", "input_text", "output_text", "message"}) -_PLAIN_KEYS: Final = frozenset({"role", "type", "text", "content", "status"}) -_TOOL_ROLES: Final = frozenset({"tool", "function"}) -_IGNORED_KEYS: Final = frozenset({"cache_control"}) -_CALL_ID_KEYS: Final = frozenset({"id", "call_id", "tool_use_id", "tool_call_id"}) -_OPAQUE_KEYS: Final = frozenset({"data", "file_data", "signature", "encrypted_content"}) - - def get_str_from_messages_with_tools(messages: object) -> str: """ - Semantic-cache prompt for chat-style ``messages``. Plain text parts stay plain text so text-only prompts embed - as they always have. Every other part (tool calls and results, images, unknown block types, extra fields) is kept - as compact JSON, so a new request shape changes the prompt instead of silently vanishing from it + ``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns + that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``) + produce different text. Each result is encoded with the position of the call it answers, since call ids are + random per session """ message_mappings: Final = tuple(_str_mappings(messages)) - call_ordinals: Final = _call_id_ordinals(message_mappings) - return "".join(_message_prompt(message, call_ordinals) for message in message_mappings) + call_ordinals: Final = tool_call_ordinals(_message_tool_call_ids(message_mappings)) + return "".join(_message_str_with_tools(message, call_ordinals) for message in message_mappings) -def get_str_from_responses_input(responses_input: object) -> str | None: - """``get_str_from_messages_with_tools`` for Responses API ``input``, one stripped line per part""" - call_ordinals: Final = _call_id_ordinals(responses_input) - prompt: Final = "\n".join(_responses_input_parts(responses_input, call_ordinals)).strip() - return prompt or None +def tool_call_str(name: object, arguments: object) -> str: + return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}' -def _message_prompt(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - role: Final = message.get("role") - if (isinstance(role, str) and role in _TOOL_ROLES) or not _is_plain(message): - return _compact_json(_normalized(message, call_ordinals)) - return _plain_prompt(message, call_ordinals) +def tool_result_str(call_id: object, call_ordinals: Mapping[str, int], output: str) -> str: + ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None + return f'{{"result_of_call":{_compact_json(ordinal)},"output":{_compact_json(output)}}}' -def _plain_prompt(value: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: - text: Final = value.get("text") - return (text if isinstance(text, str) else "") + _content_prompt(value.get("content"), call_ordinals) - - -def _content_prompt(content: object, call_ordinals: Mapping[str, int]) -> str: - value: Final = _plain_value(content) - if value is None: - return "" - if isinstance(value, str): - return value - if isinstance(value, (list, tuple)): - return "".join(_content_prompt(item, call_ordinals) for item in value) - if _is_str_mapping(value) and _is_plain(value): - return _plain_prompt(value, call_ordinals) - return _compact_json(_normalized(value, call_ordinals)) - - -def _responses_input_parts(value: object, call_ordinals: Mapping[str, int]) -> Iterator[str]: - item: Final = _plain_value(value) - if item is None: - return - if isinstance(item, str): - yield from _stripped(item) - elif isinstance(item, (list, tuple)): - for child in item: - yield from _responses_input_parts(child, call_ordinals) - elif _is_str_mapping(item) and _is_plain(item): - text: Final = item.get("text") - yield from _stripped(text if isinstance(text, str) else "") - yield from _responses_input_parts(item.get("content"), call_ordinals) - else: - yield _compact_json(_normalized(item, call_ordinals)) - - -def _stripped(text: str) -> Iterator[str]: - stripped: Final = text.strip() - if stripped: - yield stripped - - -def _is_plain(value: Mapping[str, object]) -> bool: - value_type: Final = value.get("type") - text: Final = value.get("text") - return ( - (value_type is None or (isinstance(value_type, str) and value_type in _PLAIN_TYPES)) - and (text is None or isinstance(text, str)) - and all(key in _PLAIN_KEYS or key in _CALL_ID_KEYS for key in _kept_keys(value)) - ) - - -def _kept_keys(value: Mapping[str, object]) -> Iterator[str]: - return (key for key, item in value.items() if key not in _IGNORED_KEYS and not _is_empty(item)) - - -def _normalized(value: object, call_ordinals: Mapping[str, int], key: str | None = None) -> object: - item: Final = _plain_value(value) - if _is_str_mapping(item): - return MappingProxyType({kept: _normalized(item[kept], call_ordinals, kept) for kept in _kept_keys(item)}) - if isinstance(item, (list, tuple)): - return tuple(_normalized(child, call_ordinals) for child in item) - if not isinstance(item, str): - return item - if key in _CALL_ID_KEYS: - return call_ordinals.get(item, item) - if key in _OPAQUE_KEYS or item.startswith("data:"): - return f"sha256:{hashlib.sha256(item.encode('utf-8', 'surrogatepass')).hexdigest()[:16]}" - return item - - -def _call_id_ordinals(value: object) -> Mapping[str, int]: - return MappingProxyType( - {call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(_call_ids(value)), start=1)} - ) - - -def _call_ids(value: object, key: str | None = None) -> Iterator[str]: - item: Final = _plain_value(value) - if isinstance(item, str): - if key in _CALL_ID_KEYS: - yield item - elif _is_str_mapping(item): - for child_key, child in item.items(): - if child_key not in _IGNORED_KEYS: - yield from _call_ids(child, child_key) - elif isinstance(item, (list, tuple)): - for child in item: - yield from _call_ids(child) - - -def _is_empty(value: object) -> bool: - return value is None or (isinstance(value, (str, list, tuple, Mapping)) and not value) +def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]: + string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str)) + return MappingProxyType({call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(string_ids), start=1)}) def _compact_json(value: object) -> str: - return json.dumps(value, separators=(",", ":"), default=_json_default) + return json.dumps(value, separators=(",", ":"), default=str) -def _json_default(value: object) -> object: - return dict(value) if isinstance(value, MappingProxyType) else str(value) +def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]: + for message in messages: + yield from ( + block.get("id") for block in _str_mappings(message.get("content")) if block.get("type") == "tool_use" + ) + yield from (tool_call.get("id") for tool_call in _str_mappings(message.get("tool_calls"))) -def _plain_value(value: object) -> object: - return value.model_dump() if isinstance(value, BaseModel) else value +def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + content: Final = _content_str_with_tools(message.get("content"), call_ordinals) + return ( + ( + tool_result_str(message.get("tool_call_id"), call_ordinals, content) + if message.get("role") == "tool" + else content + ) + + "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls"))) + + extract_search_results_text(message.get("search_results")) + ) + + +def _content_str_with_tools(content: object, call_ordinals: Mapping[str, int]) -> str: + if isinstance(content, str): + return content + return "".join(_block_str_with_tools(block, call_ordinals) for block in _str_mappings(content)) + + +def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str: + block_type: Final = block.get("type") + if block_type == "tool_use": + return tool_call_str(block.get("name"), block.get("input")) + if block_type == "tool_result": + return tool_result_str( + block.get("tool_use_id"), call_ordinals, _content_str_with_tools(block.get("content"), call_ordinals) + ) + text: Final = block.get("text") + return text if isinstance(text, str) else "" + + +def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str: + function: Final = _as_str_mapping(tool_call.get("function")) + if function is None: + return tool_call_str(None, None) + return tool_call_str(function.get("name"), function.get("arguments")) def _str_mappings(values: object) -> Iterator[Mapping[str, object]]: items: Final = values if isinstance(values, (list, tuple)) else () - return (mapping for item in items if _is_str_mapping(mapping := _plain_value(item))) + return (mapping for item in items if (mapping := _as_str_mapping(item)) is not None) + + +def _as_str_mapping(value: object) -> Mapping[str, object] | None: + if isinstance(value, BaseModel): + return value.model_dump() + if _is_str_mapping(value): + return value + return None def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: message and block keys are str diff --git a/tests/unit/caching/test_redis_semantic_cache.py b/tests/unit/caching/test_redis_semantic_cache.py index 6040ae2b214..3359796ec84 100644 --- a/tests/unit/caching/test_redis_semantic_cache.py +++ b/tests/unit/caching/test_redis_semantic_cache.py @@ -531,7 +531,7 @@ def test_redis_semantic_cache_get_cache_uses_responses_string_input(): ) -def test_redis_semantic_cache_set_cache_embeds_every_part_of_structured_responses_input(): +def test_redis_semantic_cache_set_cache_flattens_structured_responses_input(): from litellm.caching.redis_semantic_cache import RedisSemanticCache redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache) @@ -561,7 +561,7 @@ def test_redis_semantic_cache_set_cache_embeds_every_part_of_structured_response ) redis_semantic_cache.llmcache.store.assert_called_once_with( - 'What is the capital of France?\nAnswer briefly.\n{"type":"input_image","image_url":"https://example.com/paris.png"}', + "What is the capital of France?\nAnswer briefly.", "{'content': 'Paris'}", vector=[0.1, 0.2, 0.3], filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"}, @@ -593,11 +593,10 @@ def test_redis_semantic_cache_prompt_extraction_keeps_tool_turns_distinct(): ] assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == ( - 'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}' - '{"type":"tool_result","tool_use_id":1,"content":"ok"}' + 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1,"output":"ok"}' ) - assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) != ( - RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) + assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == ( + 'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1,"output":"ok"}' ) @@ -613,8 +612,7 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls() ) assert prompt == ( - 'update the config\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}' - '\n{"type":"function_call_output","call_id":1,"output":"ok"}' + 'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1,"output":"ok"}' ) @@ -634,13 +632,30 @@ def test_redis_semantic_cache_prompt_extraction_keeps_structured_function_call_o ] ) - assert prompt_for("wrote 5 bytes") == ( - 'write hello\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}\n' - '{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":"wrote 5 bytes"}]}' + expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}' + assert ( + prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1,"output":"wrote 5 bytes"}}' ) assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError") +def test_redis_semantic_cache_prompt_extraction_joins_multi_part_function_call_output_lines(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input=[ + {"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"}, + { + "type": "function_call_output", + "call_id": "c1", + "output": [{"type": "input_text", "text": " line one "}, {"type": "input_text", "text": "line two"}], + }, + ] + ) + + assert prompt == '{"name":"run","arguments":"{}"}\n{"result_of_call":1,"output":"line one\\nline two"}' + + def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls(): from litellm.caching.redis_semantic_cache import RedisSemanticCache @@ -655,63 +670,80 @@ def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_ans ) assert prompt_for("c2", "c1") == ( - '{"type":"function_call","call_id":1,"name":"read","arguments":"a"}\n' - '{"type":"function_call","call_id":2,"name":"read","arguments":"b"}\n' - '{"type":"function_call_output","call_id":2,"output":"empty"}\n' - '{"type":"function_call_output","call_id":1,"output":"secret"}' + '{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n' + '{"result_of_call":2,"output":"empty"}\n{"result_of_call":1,"output":"secret"}' ) assert prompt_for("c1", "c2") != prompt_for("c2", "c1") -def test_redis_semantic_cache_prompt_extraction_replaces_encrypted_reasoning_with_a_digest(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - def prompt_for(encrypted_content: str) -> str | None: - return RedisSemanticCache._get_prompt_from_kwargs( - input=[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": encrypted_content}] - ) - - blob = "gAAAAB" * 100 - prompt = prompt_for(blob) - assert prompt is not None - assert blob not in prompt - assert prompt.startswith('{"type":"reasoning","id":1,"encrypted_content":"sha256:') - assert prompt != prompt_for(blob + "x") - - def test_redis_semantic_cache_prompt_extraction_handles_model_objects(): - from pydantic import BaseModel - from litellm.caching.redis_semantic_cache import RedisSemanticCache - class ResponseInputMessage(BaseModel): - content: list[dict[str, str]] + class ModelDumpInput: + def model_dump(self): + return {"content": [{"text": "model dump prompt"}]} + + class DictInput: + def dict(self): + return {"content": [{"output_text": "dict prompt"}]} prompt = RedisSemanticCache._get_prompt_from_kwargs( input=[ - ResponseInputMessage(content=[{"type": "input_text", "text": "model dump prompt"}]), - {"content": [{"type": "output_text", "text": "inline prompt", "annotations": []}]}, + ModelDumpInput(), + DictInput(), + {"content": [{"input_text": "inline prompt"}]}, + {"content": [{"type": "input_image", "image_url": "https://example.com"}]}, ] ) - assert prompt == "model dump prompt\ninline prompt" + assert prompt == "model dump prompt\ndict prompt\ninline prompt" -def test_redis_semantic_cache_prompt_extraction_keeps_image_only_input(): - from litellm.caching.redis_semantic_cache import RedisSemanticCache - - assert RedisSemanticCache._get_prompt_from_kwargs( - input=[{"type": "input_image", "image_url": "https://example.com/a.png"}] - ) == '{"type":"input_image","image_url":"https://example.com/a.png"}' - - -def test_redis_semantic_cache_prompt_extraction_returns_none_without_input(): +def test_redis_semantic_cache_prompt_extraction_returns_none_without_text(): from litellm.caching.redis_semantic_cache import RedisSemanticCache assert RedisSemanticCache._get_prompt_from_kwargs() is None assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None - assert RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_text", "text": " "}]) is None + assert ( + RedisSemanticCache._get_prompt_from_kwargs( + input=[{"type": "input_image", "image_url": "https://example.com"}] + ) + is None + ) + + +def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + prompt = RedisSemanticCache._get_prompt_from_kwargs( + input={"text": " ", "input_text": "fallback prompt"} + ) + + assert prompt == "fallback prompt" + + +def test_redis_semantic_cache_prompt_extraction_skips_blank_object_text_keys(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + class ResponseInput: + text = " " + input_text = "fallback prompt" + + prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) + + assert prompt == "fallback prompt" + + +def test_redis_semantic_cache_prompt_extraction_handles_object_content(): + from litellm.caching.redis_semantic_cache import RedisSemanticCache + + class ResponseInput: + content = [{"text": "object content prompt"}] + + prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput()) + + assert prompt == "object content prompt" def test_redis_semantic_cache_set_cache_skips_blank_responses_input(): diff --git a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py index 468bc71f95d..59663de68d0 100644 --- a/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py +++ b/tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py @@ -3,7 +3,6 @@ import functools import json import os import sys -from collections.abc import Callable from typing import Final import pytest @@ -2024,12 +2023,12 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}], }, ], - 'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}', - id="anthropic-tool-use-with-its-call-position", + 'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}', + id="anthropic-tool-use-name-and-input-without-id", ), pytest.param( [_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}], - 'fix the failing test{"type":"tool_result","tool_use_id":1,"content":"calc.py"}', + 'fix the failing test{"result_of_call":null,"output":"calc.py"}', id="anthropic-string-tool-result", ), pytest.param( @@ -2046,7 +2045,7 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} ], } ], - '{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next', + '{"result_of_call":null,"output":"ab"}next', id="anthropic-nested-text-tool-result-then-text", ), pytest.param( @@ -2062,11 +2061,9 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} }, {"role": "tool", "tool_call_id": "c1", "content": "ok"}, ], - 'fix the failing test{"role":"assistant","content":"writing","tool_calls":[' - '{"id":1,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}},' - '{"id":2,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}}]}' - '{"role":"tool","tool_call_id":1,"content":"ok"}', - id="openai-tool-calls-then-tool-result", + 'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}' + '{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1,"output":"ok"}', + id="openai-tool-calls-in-order-before-tool-result", ), pytest.param( [ @@ -2085,9 +2082,8 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} ], }, ], - '{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}' - '{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}' - '{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}', + '{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}' + '{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}', id="anthropic-parallel-tool-results-tagged-with-their-call", ), pytest.param( @@ -2097,11 +2093,19 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} "content": None, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}], }, - {"role": "tool", "tool_call_id": "call_0", "content": "C"}, + { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, + {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}, + ], + }, + {"role": "tool", "tool_call_id": "call_1", "content": "C"}, ], - '{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}' - '{"role":"tool","tool_call_id":1,"content":"C"}', - id="null-content-dropped-from-encoded-message", + '{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}' + '{"result_of_call":2,"output":"C"}', + id="reused-call-ids-keep-first-position", ), pytest.param( [ @@ -2112,13 +2116,13 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} "search_results": [{"source": "s", "title": "t"}], } ], - '{"role":"tool","tool_call_id":1,"content":"small","search_results":[{"source":"s","title":"t"}]}', - id="tool-result-keeps-search-results", + '{"result_of_call":null,"output":"small"}st', + id="tool-result-search-results-follow-the-encoded-result", ), pytest.param( - [{"role": "tool", "content": 'x"}{"role":"tool","content":"y'}], - '{"role":"tool","content":"x\\"}{\\"role\\":\\"tool\\",\\"content\\":\\"y"}', - id="tool-output-is-escaped-even-without-a-call-id", + [{"role": "tool", "tool_call_id": "c1", "content": '"},{"result_of_call":2,"output":"'}], + '{"result_of_call":null,"output":"\\"},{\\"result_of_call\\":2,\\"output\\":\\""}', + id="tool-output-cannot-forge-an-encoded-result", ), pytest.param( [ @@ -2127,32 +2131,17 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"} tool_calls=[ChatCompletionMessageToolCall(id="c1", function=Function(name="read", arguments="{}"))], ) ], - '{"role":"assistant","tool_calls":[{"function":{"arguments":"{}","name":"read"},"id":1,"type":"function"}]}', + '{"name":"read","arguments":"{}"}', id="openai-response-message-object", ), pytest.param( - [{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}], - "hi", - id="cache-control-ignored", - ), - pytest.param( - [{"role": "user", "name": "alice", "content": "hi"}], - '{"role":"user","name":"alice","content":"hi"}', - id="extra-message-field-kept", - ), - pytest.param( - [{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}], - '{"type":"brand_new_block","payload":{"k":"v"}}!', - id="unknown-block-type-kept", - ), - pytest.param( - [{"role": "user", "content": [{"type": "text", "text": 3}]}], - '{"type":"text","text":3}', - id="non-string-text-kept", + [{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}, "junk"], + '{"name":null,"arguments":null}', + id="malformed-tool-call-entries", ), ], ) -def test_get_str_from_messages_with_tools_keeps_every_part(messages: list[object], expected: str) -> None: +def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[object], expected: str) -> None: assert get_str_from_messages_with_tools(messages) == expected @@ -2161,54 +2150,25 @@ def test_get_str_from_messages_with_tools_keeps_every_part(messages: list[object [ pytest.param([_TASK, {"role": "assistant", "content": "done"}], id="string-content"), pytest.param( - [{"role": "user", "content": [{"type": "text", "text": "what is "}, {"type": "text", "text": "this"}]}], - id="text-parts", + [ + { + "role": "user", + "content": [ + {"type": "text", "text": "what is "}, + {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, + {"type": "text", "text": "this"}, + ], + } + ], + id="text-and-image-parts", ), pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"), ], ) -def test_get_str_from_messages_with_tools_matches_get_str_from_messages_for_text(messages: list[object]) -> None: +def test_get_str_from_messages_with_tools_matches_get_str_from_messages_without_tools(messages: list[object]) -> None: assert get_str_from_messages_with_tools(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures -def _anthropic_image(data: str) -> list[object]: - return [{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "data": data}}]}] - - -def _openai_image(url: str) -> list[object]: - return [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": url}}]}] - - -def _thinking(signature: str) -> list[object]: - return [{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": signature}]}] - - -@pytest.mark.parametrize( - ("build", "blob"), - [ - pytest.param(_anthropic_image, "iVBORw0KGgo" * 50, id="anthropic-base64-image"), - pytest.param(_openai_image, "data:image/png;base64," + "iVBORw0KGgo" * 50, id="openai-data-url-image"), - pytest.param(_thinking, "EqQBCkgIARABGAIiQL" * 20, id="thinking-signature"), - ], -) -def test_get_str_from_messages_with_tools_replaces_opaque_blobs_with_distinct_digests( - build: Callable[[str], list[object]], blob: str -) -> None: - prompt: Final = get_str_from_messages_with_tools(build(blob)) - - assert blob not in prompt - assert "sha256:" in prompt - assert len(prompt) < len(blob) - assert prompt == get_str_from_messages_with_tools(build(blob)) - assert prompt != get_str_from_messages_with_tools(build(blob + "A")) - - -def test_get_str_from_messages_with_tools_keeps_image_urls_distinct() -> None: - assert get_str_from_messages_with_tools(_openai_image("https://example.com/a.png")) != ( - get_str_from_messages_with_tools(_openai_image("https://example.com/b.png")) - ) - - def _parallel_reads(result_for_a: str, result_for_b: str, *, call_id_prefix: str = "c") -> list[object]: return [ {