scriptoriumadmin/litellm
1
0
Fork 0
mirror of https://github.com/BerriAI/litellm.git synced 2026-10-08 03:08:45 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 115

feat(caching): embed every semantic cache prompt field except volatile ones

Browse source 
Replace the per-shape allowlist in the Python and Rust semantic cache prompt
walkers with one include-by-default walker. Plain text keeps its old
concatenation; any other block or message is embedded as compact JSON with
call ids mapped to ordinals, cache_control dropped, and signatures, encrypted
content and base64 data replaced with a short sha256 digest.

Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry 2026-09-30 23:09:15 +00:00
parent 127506254b
commit 5aed3ab3de
9 changed files with 606 additions and 739 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
litellm-rust/Cargo.lock generated
View file

@ -3426,6 +3426,7 @@ dependencies = [
"rstest",
"serde",
"serde_json",
"sha2 0.10.9",
"thiserror 2.0.19",
"tokio",
]

8
litellm-rust/crates/cache-qdrant-semantic/tests/qdrant.rs
View file

@ -225,7 +225,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) {
}
#[rstest]
#[case::content_parts_skip_images(
#[case::content_parts_keep_images_in_place(
json!([
{"role": "user", "content": "hello"},
{
@ -237,9 +237,9 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) {
],
},
]),
"helloworld!"
r#"helloworld{"type":"image_url","image_url":{"url":"ignored"}}!"#
)]
#[case::search_results_and_compact_citations(
#[case::tool_message_with_search_results_kept_as_json(
json!([{
"role": "tool",
"content": null,
@ -250,7 +250,7 @@ async fn async_and_sync_set_get_store_exact_payload(entry: JsonValue) {
"citations": {"page": 1, "section": "intro"},
}],
}]),
r#"{"result_of_call":null,"output":""}sourcetitlebody{"page":1,"section":"intro"}"#
r#"{"role":"tool","search_results":[{"source":"source","title":"title","content":[{"text":"body"}],"citations":{"page":1,"section":"intro"}}]}"#
)]
#[tokio::test(flavor = "multi_thread")]
async fn prompt_matches_python_message_rules(

1
litellm-rust/crates/cache/Cargo.toml vendored
View file

@ -8,6 +8,7 @@ repository.workspace = true
[dependencies]
serde.workspace = true
serde_json = { workspace = true, features = ["preserve_order"] }
sha2.workspace = true
thiserror.workspace = true
[dev-dependencies]

400
litellm-rust/crates/cache/src/semantic.rs vendored
View file

@ -8,9 +8,10 @@ use std::{collections::HashMap, future::Future, io};
use serde::Serialize;
use serde_json::{
Value, json,
Map, Value,
ser::{CharEscape, Formatter, Serializer},
};
use sha2::{Digest, Sha256};
use crate::{BaseCache, Error, SemanticCacheContext};
@ -83,122 +84,225 @@ impl Embedder for PreparedEmbedding {
}
}
/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results,
/// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the
/// position of the call it answers.
const PLAIN_TYPES: [&str; 4] = ["text", "input_text", "output_text", "message"];
const PLAIN_KEYS: [&str; 5] = ["role", "type", "text", "content", "status"];
const TOOL_ROLES: [&str; 2] = ["tool", "function"];
const IGNORED_KEYS: [&str; 1] = ["cache_control"];
const CALL_ID_KEYS: [&str; 4] = ["id", "call_id", "tool_use_id", "tool_call_id"];
const OPAQUE_KEYS: [&str; 4] = ["data", "file_data", "signature", "encrypted_content"];
/// `get_str_from_messages_with_tools`: plain text parts stay plain text, and every other part is
/// kept as compact JSON with call ids replaced by their position and opaque blobs by a digest.
pub fn str_from_messages(messages: &[Value]) -> String {
let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect();
let call_ordinals = tool_call_ordinals(messages.iter().flat_map(|message| {
let block_ids = message
.get("content")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter(|block| block.get("type").and_then(Value::as_str) == Some("tool_use"))
.filter_map(|block| block.get("id"));
let tool_call_ids = message
.get("tool_calls")
.and_then(Value::as_array)
.into_iter()
.flatten()
.filter_map(|tool_call| tool_call.get("id"));
block_ids.chain(tool_call_ids)
}));
let mut text = String::new();
for message in messages {
if message.get("role").and_then(Value::as_str) == Some("tool") {
let mut output = String::new();
push_content_text(&mut output, message.get("content"), &call_ordinals);
text.push_str(&tool_result_json(
message.get("tool_call_id"),
&call_ordinals,
&output,
));
} else {
push_content_text(&mut text, message.get("content"), &call_ordinals);
}
if let Some(Value::Array(tool_calls)) = message.get("tool_calls") {
for tool_call in tool_calls.iter().filter_map(Value::as_object) {
let function = tool_call.get("function");
text.push_str(&tool_call_json(
function.and_then(|function| function.get("name")),
function.and_then(|function| function.get("arguments")),
));
}
}
push_search_results_text(&mut text, message.get("search_results"));
}
text
let messages: Vec<&Value> = messages
.iter()
.filter(|message| message.is_object())
.collect();
let call_ordinals = call_id_ordinals(messages.iter().copied());
messages
.into_iter()
.filter_map(Value::as_object)
.map(|message| message_prompt(message, &call_ordinals))
.collect()
}
/// `_content_str_with_tools`: text parts, Anthropic `tool_use` blocks and `tool_result` content.
fn push_content_text(
text: &mut String,
/// `get_str_from_responses_input`: `str_from_messages` for a Responses API `input`, one stripped
/// line per part. `None` when nothing is left.
fn str_from_responses_input(input: &Value) -> Option<String> {
let call_ordinals = call_id_ordinals(std::iter::once(input));
let mut parts = Vec::new();
push_responses_input_parts(input, &call_ordinals, &mut parts);
let prompt = python_strip(&parts.join("\n")).to_owned();
(!prompt.is_empty()).then_some(prompt)
}
fn message_prompt(message: &Map<String, Value>, call_ordinals: &HashMap<&str, usize>) -> String {
let is_tool = message
.get("role")
.and_then(Value::as_str)
.is_some_and(|role| TOOL_ROLES.contains(&role));
if is_tool || !is_plain(message) {
return compact_json(&normalized(message, call_ordinals));
}
plain_prompt(message, call_ordinals)
}
fn plain_prompt(value: &Map<String, Value>, call_ordinals: &HashMap<&str, usize>) -> String {
let mut prompt = value
.get("text")
.and_then(Value::as_str)
.unwrap_or_default()
.to_owned();
push_content_prompt(&mut prompt, value.get("content"), call_ordinals);
prompt
}
fn push_content_prompt(
prompt: &mut String,
content: Option<&Value>,
call_ordinals: &HashMap<&str, usize>,
) {
match content {
Some(Value::String(content)) => text.push_str(content),
Some(Value::Array(blocks)) => {
for block in blocks.iter().filter_map(Value::as_object) {
match block.get("type").and_then(Value::as_str) {
Some("tool_use") => {
text.push_str(&tool_call_json(block.get("name"), block.get("input")));
}
Some("tool_result") => {
let mut output = String::new();
push_content_text(&mut output, block.get("content"), call_ordinals);
text.push_str(&tool_result_json(
block.get("tool_use_id"),
call_ordinals,
&output,
));
}
_ => {
if let Some(block_text) = block.get("text").and_then(Value::as_str) {
text.push_str(block_text);
}
}
}
None | Some(Value::Null) => {}
Some(Value::String(text)) => prompt.push_str(text),
Some(Value::Array(items)) => {
for item in items {
push_content_prompt(prompt, Some(item), call_ordinals);
}
}
_ => {}
Some(Value::Object(map)) if is_plain(map) => {
prompt.push_str(&plain_prompt(map, call_ordinals));
}
Some(Value::Object(map)) => prompt.push_str(&compact_json(&normalized(map, call_ordinals))),
Some(other) => prompt.push_str(&compact_json(other)),
}
}
/// `tool_call_ordinals`: the 1-based position of each distinct string call id, first seen first.
fn tool_call_ordinals<'a>(call_ids: impl Iterator<Item = &'a Value>) -> HashMap<&'a str, usize> {
fn push_responses_input_parts(
value: &Value,
call_ordinals: &HashMap<&str, usize>,
parts: &mut Vec<String>,
) {
match value {
Value::Null => {}
Value::String(text) => push_stripped(text, parts),
Value::Array(items) => {
for item in items {
push_responses_input_parts(item, call_ordinals, parts);
}
}
Value::Object(map) if is_plain(map) => {
push_stripped(
map.get("text").and_then(Value::as_str).unwrap_or_default(),
parts,
);
if let Some(content) = map.get("content") {
push_responses_input_parts(content, call_ordinals, parts);
}
}
Value::Object(map) => parts.push(compact_json(&normalized(map, call_ordinals))),
other => parts.push(compact_json(other)),
}
}
fn push_stripped(text: &str, parts: &mut Vec<String>) {
let stripped = python_strip(text);
if !stripped.is_empty() {
parts.push(stripped.to_owned());
}
}
/// `_is_plain`: a text-only part whose every kept key is a plain or call id key.
fn is_plain(value: &Map<String, Value>) -> bool {
let plain_type = match value.get("type") {
None | Some(Value::Null) => true,
Some(Value::String(value_type)) => PLAIN_TYPES.contains(&value_type.as_str()),
Some(_) => false,
};
let plain_text = matches!(
value.get("text"),
None | Some(Value::Null | Value::String(_))
);
plain_type
&& plain_text
&& kept_entries(value).all(|(key, _)| {
PLAIN_KEYS.contains(&key.as_str()) || CALL_ID_KEYS.contains(&key.as_str())
})
}
fn kept_entries(value: &Map<String, Value>) -> impl Iterator<Item = (&String, &Value)> {
value
.iter()
.filter(|(key, item)| !IGNORED_KEYS.contains(&key.as_str()) && !is_empty(item))
}
fn is_empty(value: &Value) -> bool {
match value {
Value::Null => true,
Value::String(text) => text.is_empty(),
Value::Array(items) => items.is_empty(),
Value::Object(map) => map.is_empty(),
Value::Bool(_) | Value::Number(_) => false,
}
}
/// `_normalized`: the part minus ignored and empty entries, with call ids replaced by their
/// position and opaque blobs by `sha256:` plus the first 16 hex digits of their digest.
fn normalized(value: &Map<String, Value>, call_ordinals: &HashMap<&str, usize>) -> Value {
Value::Object(
kept_entries(value)
.map(|(key, item)| {
(
key.clone(),
normalized_value(item, call_ordinals, Some(key)),
)
})
.collect(),
)
}
fn normalized_value(
value: &Value,
call_ordinals: &HashMap<&str, usize>,
key: Option<&str>,
) -> Value {
match value {
Value::Object(map) => normalized(map, call_ordinals),
Value::Array(items) => Value::Array(
items
.iter()
.map(|item| normalized_value(item, call_ordinals, None))
.collect(),
),
Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => call_ordinals
.get(text.as_str())
.map_or_else(|| value.clone(), |ordinal| Value::from(*ordinal)),
Value::String(text)
if key.is_some_and(|key| OPAQUE_KEYS.contains(&key)) || text.starts_with("data:") =>
{
Value::String(digest(text))
}
other => other.clone(),
}
}
fn digest(text: &str) -> String {
let hash = Sha256::digest(text.as_bytes());
let hex: String = hash[..8].iter().map(|byte| format!("{byte:02x}")).collect();
format!("sha256:{hex}")
}
/// `_call_id_ordinals`: the 1-based position of each distinct call id string, first seen first.
fn call_id_ordinals<'a>(values: impl Iterator<Item = &'a Value>) -> HashMap<&'a str, usize> {
let mut ids = Vec::new();
for value in values {
collect_call_ids(value, None, &mut ids);
}
let mut ordinals = HashMap::new();
for call_id in call_ids.filter_map(Value::as_str) {
for id in ids {
let next = ordinals.len() + 1;
ordinals.entry(call_id).or_insert(next);
ordinals.entry(id).or_insert(next);
}
ordinals
}
/// `tool_result_str`: `{"result_of_call":N,"output":...}`, with a `null` position when the result
/// answers no known call.
fn tool_result_json(
call_id: Option<&Value>,
call_ordinals: &HashMap<&str, usize>,
output: &str,
) -> String {
let ordinal = call_id
.and_then(Value::as_str)
.and_then(|call_id| call_ordinals.get(call_id));
format!(
"{{\"result_of_call\":{},\"output\":{}}}",
compact_json(&json!(ordinal)),
compact_json(&Value::String(output.to_owned())),
)
}
/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes.
fn tool_call_json(name: Option<&Value>, arguments: Option<&Value>) -> String {
compact_json(&json!({
"name": name.unwrap_or(&Value::Null),
"arguments": arguments.unwrap_or(&Value::Null),
}))
fn collect_call_ids<'a>(value: &'a Value, key: Option<&str>, ids: &mut Vec<&'a str>) {
match value {
Value::String(text) if key.is_some_and(|key| CALL_ID_KEYS.contains(&key)) => ids.push(text),
Value::Object(map) => {
for (child_key, child) in map {
if !IGNORED_KEYS.contains(&child_key.as_str()) {
collect_call_ids(child, Some(child_key), ids);
}
}
}
Value::Array(items) => {
for item in items {
collect_call_ids(item, None, ids);
}
}
_ => {}
}
}
/// The messages prompt Qdrant embeds: `None` when the request carries no messages.
@ -215,103 +319,7 @@ pub fn prompt_from_context(context: &SemanticCacheContext) -> Option<String> {
{
return Some(str_from_messages(messages));
}
let input = context.input.as_ref()?;
let call_ordinals = tool_call_ordinals(
input
.as_array()
.into_iter()
.flatten()
.filter(|item| item.get("type").and_then(Value::as_str) == Some("function_call"))
.filter_map(|item| item.get("call_id")),
);
let mut parts = Vec::new();
collect_input_text(input, &mut parts, &call_ordinals);
let prompt = python_strip(&parts.join("\n")).to_owned();
(!prompt.is_empty()).then_some(prompt)
}
/// `extract_search_results_text`.
fn push_search_results_text(text: &mut String, search_results: Option<&Value>) {
let Some(Value::Array(results)) = search_results else {
return;
};
for result in results.iter().filter_map(Value::as_object) {
for key in ["source", "title"] {
if let Some(value) = result.get(key).and_then(Value::as_str) {
text.push_str(value);
}
}
if let Some(Value::Array(content)) = result.get("content") {
for block in content.iter().filter_map(Value::as_object) {
if let Some(value) = block.get("text").and_then(Value::as_str) {
text.push_str(value);
}
}
}
if let Some(citations) = result.get("citations").filter(|value| !value.is_null()) {
text.push_str(&compact_json(citations));
}
}
}
fn collect_input_text(
value: &Value,
parts: &mut Vec<String>,
call_ordinals: &HashMap<&str, usize>,
) {
match value {
Value::String(text) => {
push_trimmed(text, parts);
}
Value::Array(items) => {
for item in items {
collect_input_text(item, parts, call_ordinals);
}
}
Value::Object(map) => {
if map.get("type").and_then(Value::as_str) == Some("function_call") {
parts.push(tool_call_json(map.get("name"), map.get("arguments")));
return;
}
if map.get("type").and_then(Value::as_str) == Some("function_call_output") {
let mut output_parts = Vec::new();
if let Some(output) = map.get("output") {
collect_input_text(output, &mut output_parts, call_ordinals);
}
parts.push(tool_result_json(
map.get("call_id"),
call_ordinals,
python_strip(&output_parts.join("\n")),
));
return;
}
if let Some(content) = map.get("content").filter(|content| !content.is_null()) {
collect_input_text(content, parts, call_ordinals);
return;
}
for key in ["text", "output", "input_text", "output_text"] {
match map.get(key) {
Some(nested @ Value::Array(_)) => {
collect_input_text(nested, parts, call_ordinals);
return;
}
Some(Value::String(text)) if push_trimmed(text, parts) => return,
_ => {}
}
}
}
_ => {}
}
}
/// Pushes `text` stripped as Python's `str.strip` does, reporting whether anything was left.
fn push_trimmed(text: &str, parts: &mut Vec<String>) -> bool {
let trimmed = python_strip(text);
if trimmed.is_empty() {
return false;
}
parts.push(trimmed.to_owned());
true
str_from_responses_input(context.input.as_ref()?)
}
/// `str.strip()`: Python's whitespace also covers the ASCII information separators.

407
litellm-rust/crates/cache/tests/semantic.rs vendored
View file

@ -16,168 +16,115 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
}
#[rstest]
#[case::empty(json!([]), "")]
#[case::string_content(json!([{"role": "user", "content": "hello"}]), "hello")]
#[case::empty(r#"[]"#, r#""#)]
#[case::string_content(r#"[{"role": "user", "content": "hello"}]"#, r#"hello"#)]
#[case::concatenates_messages(
json!([{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]),
"be brief. hello",
r#"[{"role": "system", "content": "be brief. "}, {"role": "user", "content": "hello"}]"#,
r#"be brief. hello"#
)]
#[case::text_parts(
json!([{"role": "user", "content": [
{"type": "text", "text": "What is "},
{"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
{"type": "text", "text": "this?"},
]}]),
"What is this?",
r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "text", "text": "this?"}]}]"#,
r#"What is this?"#,
)]
#[case::image_part_kept_in_place(
r#"[{"role": "user", "content": [{"type": "text", "text": "What is "}, {"type": "image_url", "image_url": {"url": "https://example.com/a.png"}}, {"type": "text", "text": "this?"}]}]"#,
r#"What is {"type":"image_url","image_url":{"url":"https://example.com/a.png"}}this?"#,
)]
#[case::missing_null_and_empty_content(
json!([{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]),
"",
r#"[{"role": "assistant"}, {"role": "assistant", "content": null}, {"role": "user", "content": ""}]"#,
r#""#,
)]
#[case::search_results_hidden_behind_small_content(
json!([{"role": "tool", "content": "small", "search_results": [
{"source": "s", "title": "t", "content": [{"text": "hidden payload"}]},
]}]),
r#"{"result_of_call":null,"output":"small"}sthidden payload"#,
#[case::non_object_messages_skipped(
r#"["junk", 3, {"role": "user", "content": "kept"}]"#,
r#"kept"#
)]
#[case::title_only_search_result(
json!([{"role": "tool", "content": "small", "search_results": [
{"source": "s", "title": "long title", "content": []},
]}]),
r#"{"result_of_call":null,"output":"small"}slong title"#,
#[case::cache_control_ignored(
r#"[{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}]"#,
r#"hi"#,
)]
#[case::search_results_without_content(
json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]),
r#"{"result_of_call":null,"output":""}st"#,
#[case::extra_message_field_kept(
r#"[{"role": "user", "name": "alice", "content": "hi"}]"#,
r#"{"role":"user","name":"alice","content":"hi"}"#
)]
#[case::search_result_fields_in_python_order(
json!([{"role": "tool", "content": "c", "search_results": [
{"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"},
{"source": "s2"},
]}]),
r#"{"result_of_call":null,"output":"c"}stbody{"enabled":true}s2"#,
#[case::unknown_block_type_kept(
r#"[{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}]"#,
r#"{"type":"brand_new_block","payload":{"k":"v"}}!"#,
)]
#[case::null_citations_skipped(
json!([{"role": "tool", "content": "c", "search_results": [
{"source": "s", "citations": null},
]}]),
r#"{"result_of_call":null,"output":"c"}s"#,
#[case::non_string_text_kept(
r#"[{"role": "user", "content": [{"type": "text", "text": 3}]}]"#,
r#"{"type":"text","text":3}"#
)]
#[case::non_string_and_non_object_entries_skipped(
json!([{"role": "tool", "content": "c", "search_results": [
"junk",
{"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]},
]}]),
r#"{"result_of_call":null,"output":"c"}kept"#,
#[case::scalar_content_kept(
r#"[{"role": "user", "content": 7}, {"role": "user", "content": [1.5, true, "x"]}]"#,
r#"71.5truex"#
)]
#[case::non_list_search_results_skipped(
json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]),
r#"{"result_of_call":null,"output":"c"}"#,
#[case::search_results_kept(
r#"[{"role": "tool", "content": "small", "search_results": [{"source": "s", "title": "t", "content": [{"type": "text", "text": "hidden payload"}], "citations": {"enabled": true}}]}]"#,
r#"{"role":"tool","content":"small","search_results":[{"source":"s","title":"t","content":[{"type":"text","text":"hidden payload"}],"citations":{"enabled":true}}]}"#,
)]
#[case::citations_compact_in_insertion_order(
json!([{"role": "tool", "search_results": [
{"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}},
]}]),
r#"{"result_of_call":null,"output":""}{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#,
)]
#[case::citations_ensure_ascii(
json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]),
r#"{"result_of_call":null,"output":""}["caf\u00e9","\u4e2d"]"#,
)]
#[case::citations_astral_chars_as_surrogate_pairs(
json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]),
r#"{"result_of_call":null,"output":""}"\ud83d\ude00""#,
)]
#[case::citations_escapes(
json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]),
r#"{"result_of_call":null,"output":""}"q\"\\\n\t\u0001/""#,
)]
#[case::citations_large_float_exponent(
json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]),
r#"{"result_of_call":null,"output":""}[1e+20,1.0]"#,
)]
#[case::citations_scalars(
json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]),
r#"{"result_of_call":null,"output":""}false3"#,
)]
#[case::anthropic_tool_use_name_and_input_without_id(
json!([
{"role": "user", "content": "fix the failing test"},
{"role": "assistant", "content": [
{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}},
]},
]),
r#"fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}"#,
#[case::anthropic_tool_use(
r#"[{"role": "user", "content": "fix the failing test"}, {"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}]}]"#,
r#"fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}"#,
)]
#[case::anthropic_string_tool_result(
json!([{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"},
]}]),
r#"{"result_of_call":null,"output":"calc.py"}"#,
r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}]"#,
r#"{"type":"tool_result","tool_use_id":1,"content":"calc.py"}"#,
)]
#[case::anthropic_nested_text_tool_result_then_text(
json!([{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "t1", "content": [
{"type": "text", "text": "a"},
{"type": "text", "text": "b"},
]},
{"type": "text", "text": "next"},
]}]),
r#"{"result_of_call":null,"output":"ab"}next"#,
r#"[{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": [{"type": "text", "text": "a"}, {"type": "text", "text": "b"}]}, {"type": "text", "text": "next"}]}]"#,
r#"{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next"#,
)]
#[case::openai_tool_calls_in_order_before_tool_result(
json!([
{"role": "assistant", "content": "writing", "tool_calls": [
{"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}},
{"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}},
]},
{"role": "tool", "tool_call_id": "c1", "content": "ok"},
]),
r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1,"output":"ok"}"#,
#[case::openai_tool_calls_then_tool_result(
r#"[{"role": "assistant", "content": "writing", "tool_calls": [{"id": "c1", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"a\"}"}}, {"id": "c2", "type": "function", "function": {"name": "write", "arguments": "{\"path\": \"b\"}"}}]}, {"role": "tool", "tool_call_id": "c1", "content": "ok"}]"#,
r#"{"role":"assistant","content":"writing","tool_calls":[{"id":1,"type":"function","function":{"name":"write","arguments":"{\"path\": \"a\"}"}},{"id":2,"type":"function","function":{"name":"write","arguments":"{\"path\": \"b\"}"}}]}{"role":"tool","tool_call_id":1,"content":"ok"}"#,
)]
#[case::anthropic_parallel_tool_results_tagged_with_their_call(
json!([
{"role": "assistant", "content": [
{"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}},
{"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}},
]},
{"role": "user", "content": [
{"type": "tool_result", "tool_use_id": "t2", "content": "B"},
{"type": "tool_result", "tool_use_id": "t1", "content": "A"},
]},
]),
r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}"#,
r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t1", "name": "Read", "input": {"path": "a"}}, {"type": "tool_use", "id": "t2", "name": "Read", "input": {"path": "b"}}]}, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t2", "content": "B"}, {"type": "tool_result", "tool_use_id": "t1", "content": "A"}]}]"#,
r#"{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}"#,
)]
#[case::reused_call_ids_keep_first_position(
json!([
{"role": "assistant", "content": null, "tool_calls": [
{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}},
]},
{"role": "assistant", "content": null, "tool_calls": [
{"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}},
{"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}},
]},
{"role": "tool", "tool_call_id": "call_1", "content": "C"},
]),
r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2,"output":"C"}"#,
r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}]}, {"role": "assistant", "content": null, "tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}}, {"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}}]}, {"role": "tool", "tool_call_id": "call_1", "content": "C"}]"#,
r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"b","arguments":"{}"}},{"id":2,"type":"function","function":{"name":"c","arguments":"{}"}}]}{"role":"tool","tool_call_id":2,"content":"C"}"#,
)]
#[case::unknown_call_ids_encode_a_null_position(
json!([
{"role": "tool", "tool_call_id": "c9", "content": "ok"},
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]},
]),
r#"{"result_of_call":null,"output":"ok"}{"result_of_call":null,"output":"done"}"#,
)]
#[case::tool_output_cannot_forge_an_encoded_result(
json!([{"role": "tool", "tool_call_id": "c1", "content": "\"},{\"result_of_call\":2,\"output\":\""}]),
r#"{"result_of_call":null,"output":"\"},{\"result_of_call\":2,\"output\":\""}"#,
#[case::tool_output_cannot_forge_a_record(
r#"[{"role": "tool", "content": "x\"}{\"role\":\"tool\",\"content\":\"y"}]"#,
r#"{"role":"tool","content":"x\"}{\"role\":\"tool\",\"content\":\"y"}"#
)]
#[case::malformed_tool_call_entries(
json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]),
r#"{"name":null,"arguments":null}"#,
r#"[{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]"#,
r#"{"role":"assistant","tool_calls":[{"id":1,"type":"function"},"junk"]}"#,
)]
fn str_from_messages_matches_python(#[case] messages: Value, #[case] expected: &str) {
assert_eq!(str_from_messages(messages.as_array().unwrap()), expected);
#[case::anthropic_base64_image_digested(
r#"[{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "media_type": "image/png", "data": "iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#,
r#"{"type":"image","source":{"type":"base64","media_type":"image/png","data":"sha256:b1a66f8de3276946"}}"#,
)]
#[case::openai_data_url_digested(
r#"[{"role": "user", "content": [{"type": "image_url", "image_url": {"url": "data:image/png;base64,iVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgoiVBORw0KGgo"}}]}]"#,
r#"{"type":"image_url","image_url":{"url":"sha256:19c7f028a60a33b8"}}"#,
)]
#[case::thinking_signature_digested(
r#"[{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": "EqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARABEqQBCkgIARAB"}, {"type": "text", "text": "done"}]}]"#,
r#"{"type":"thinking","thinking":"hmm","signature":"sha256:b20cbbd82e4b3791"}done"#,
)]
#[case::non_ascii_escaped(
r#"[{"role": "tool", "content": "caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}]"#,
r#"{"role":"tool","content":"caf\u00e9 \u4e2d \ud83d\ude00 \u007f q\"\\\n\t\u0001/"}"#
)]
#[case::non_ascii_plain_text_kept_raw(
r#"[{"role": "user", "content": "caf\u00e9 \ud83d\ude00"}]"#,
r#"café 😀"#
)]
#[case::floats_in_python_repr(
r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {"a": [1e+20, 1.0, 1.5e-07, 0.0001, 1e+16, -2.5, 3]}}]}]"#,
r#"{"type":"tool_use","id":1,"name":"n","input":{"a":[1e+20,1.0,1.5e-07,0.0001,1e+16,-2.5,3]}}"#,
)]
#[case::empty_fields_dropped(
r#"[{"role": "assistant", "content": [{"type": "tool_use", "id": "t", "name": "n", "input": {}, "extra": [], "note": ""}]}]"#,
r#"{"type":"tool_use","id":1,"name":"n"}"#,
)]
fn str_from_messages_matches_python(#[case] messages: &str, #[case] expected: &str) {
let messages: Vec<Value> = serde_json::from_str(messages).unwrap();
assert_eq!(str_from_messages(&messages), expected);
}
#[rstest]
@ -196,167 +143,109 @@ fn prompt_from_messages_reads_messages_only(
#[rstest]
#[case::prefers_messages(
Some(json!([{"content": "message prompt"}])),
Some(json!("responses prompt")),
Some("message prompt"),
Some(r#"[{"content": "message prompt"}]"#),
Some(r#""responses prompt""#),
Some(r#"message prompt"#)
)]
#[case::empty_messages_fall_back_to_input(
Some(json!([])),
Some(json!("responses prompt")),
Some("responses prompt"),
Some(r#"[]"#),
Some(r#""responses prompt""#),
Some(r#"responses prompt"#)
)]
#[case::messages_without_text_keep_an_empty_prompt(
Some(json!([{"content": null}])),
Some(json!("x")),
Some(""),
Some(r#"[{"content": null}]"#),
Some(r#""x""#),
Some(r#""#)
)]
#[case::nothing(None, None, None)]
#[case::null_input(None, Some(Value::Null), None)]
#[case::blank_string(None, Some(json!(" ")), None)]
#[case::null_input(None, None, None)]
#[case::blank_string(None, Some(r#"" ""#), None)]
#[case::trimmed_string(
None,
Some(json!(" What is the capital of France?\n")),
Some("What is the capital of France?"),
Some(r#"" What is the capital of France?\n""#),
Some(r#"What is the capital of France?"#)
)]
#[case::image_only(
None,
Some(json!([{"type": "input_image", "image_url": "https://example.com"}])),
#[case::strip_information_separators(None, Some(r#""\u001ca\u001f""#), Some(r#"a"#))]
#[case::image_only_kept(
None,
Some(r#"[{"type": "input_image", "image_url": "https://example.com"}]"#),
Some(r#"{"type":"input_image","image_url":"https://example.com"}"#)
)]
#[case::structured_input(
None,
Some(json!([{"role": "user", "content": [
{"type": "input_text", "text": "What is the capital of France?"},
{"type": "input_text", "text": "Answer briefly."},
{"type": "input_image", "image_url": "https://example.com/paris.png"},
]}])),
Some("What is the capital of France?\nAnswer briefly."),
Some(r#"[{"role": "user", "content": [{"type": "input_text", "text": "What is the capital of France?"}, {"type": "input_text", "text": "Answer briefly."}, {"type": "input_image", "image_url": "https://example.com/paris.png"}]}]"#),
Some(r#"What is the capital of France?
Answer briefly.
{"type":"input_image","image_url":"https://example.com/paris.png"}"#),
)]
#[case::model_objects_after_dump(
#[case::message_items_with_ids(
None,
Some(json!([
{"content": [{"text": "model dump prompt"}]},
{"content": [{"output_text": "dict prompt"}]},
{"content": [{"input_text": "inline prompt"}]},
{"content": [{"type": "input_image", "image_url": "https://example.com"}]},
])),
Some("model dump prompt\ndict prompt\ninline prompt"),
Some(r#"[{"type": "message", "id": "msg_1", "role": "assistant", "status": "completed", "content": [{"type": "output_text", "text": "hi", "annotations": []}]}]"#),
Some(r#"hi"#),
)]
#[case::object_content(
None,
Some(json!({"content": [{"text": "object content prompt"}]})),
Some("object content prompt"),
Some(r#"{"content": [{"text": "object content prompt"}]}"#),
Some(r#"object content prompt"#)
)]
#[case::string_content(None, Some(json!({"content": " inline "})), Some("inline"))]
#[case::null_content_uses_text_keys(
#[case::string_content(None, Some(r#"{"content": " inline "}"#), Some(r#"inline"#))]
#[case::unknown_keys_kept(
None,
Some(json!({"content": null, "output": "tool output"})),
Some("tool output"),
Some(r#"{"text": " ", "input_text": "fallback prompt"}"#),
Some(r#"{"text":" ","input_text":"fallback prompt"}"#)
)]
#[case::content_wins_over_text(None, Some(json!({"content": [], "text": "ignored"})), None)]
#[case::text_key_precedence(
#[case::nested_lists(
None,
Some(json!({"output_text": "d", "input_text": "c", "output": "b", "text": "a"})),
Some("a"),
Some(r#"[["a", [" b "]], "", "c"]"#),
Some(
r#"a
b
c"#
)
)]
#[case::input_text_key(None, Some(json!({"input_text": "only input"})), Some("only input"))]
#[case::output_text_key(None, Some(json!({"output_text": "only output"})), Some("only output"))]
#[case::non_string_text_keys_skipped(
#[case::scalars_kept(
None,
Some(json!({"text": 1, "output": "fallback"})),
Some("fallback"),
Some(r#"[1, true, null, "kept"]"#),
Some(
r#"1
true
kept"#
)
)]
#[case::nested_lists(None, Some(json!([["a", [" b "]], "", "c"])), Some("a\nb\nc"))]
#[case::scalars_ignored(None, Some(json!([1, true, null, "kept"])), Some("kept"))]
#[case::blank_text_parts(None, Some(r#"[{"type": "input_text", "text": " "}]"#), None)]
#[case::responses_function_call(
None,
Some(json!([
{"role": "user", "content": "update the config"},
{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"},
{"type": "function_call_output", "call_id": "c1", "output": "ok"},
])),
Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1,\"output\":\"ok\"}"),
Some(r#"[{"role": "user", "content": "update the config"}, {"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"}, {"type": "function_call_output", "call_id": "c1", "output": "ok"}]"#),
Some(r#"update the config
{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\"path\":\"a.yaml\"}"}
{"type":"function_call_output","call_id":1,"output":"ok"}"#),
)]
#[case::responses_structured_function_call_output(
None,
Some(json!([
{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]},
])),
Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1,\"output\":\"denied\"}"),
)]
#[case::responses_multi_part_output_joined_by_lines(
None,
Some(json!([
{"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"},
{"type": "function_call_output", "call_id": "c1", "output": [
{"type": "input_text", "text": " line one "},
{"type": "input_text", "text": "line two"},
]},
])),
Some(r#"{"name":"run","arguments":"{}"}
{"result_of_call":1,"output":"line one\nline two"}"#),
Some(r#"[{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"}, {"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": " denied "}]}]"#),
Some(r#"{"type":"function_call","call_id":1,"name":"write_file","arguments":"{}"}
{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":" denied "}]}"#),
)]
#[case::responses_parallel_outputs_tagged_with_their_call(
None,
Some(json!([
{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"},
{"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"},
{"type": "function_call_output", "call_id": "c2", "output": "B"},
{"type": "function_call_output", "call_id": "c1", "output": "A"},
])),
Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2,\"output\":\"B\"}\n{\"result_of_call\":1,\"output\":\"A\"}"),
Some(r#"[{"type": "function_call", "call_id": "c1", "name": "read", "arguments": "a"}, {"type": "function_call", "call_id": "c2", "name": "read", "arguments": "b"}, {"type": "function_call_output", "call_id": "c2", "output": "B"}, {"type": "function_call_output", "call_id": "c1", "output": "A"}]"#),
Some(r#"{"type":"function_call","call_id":1,"name":"read","arguments":"a"}
{"type":"function_call","call_id":2,"name":"read","arguments":"b"}
{"type":"function_call_output","call_id":2,"output":"B"}
{"type":"function_call_output","call_id":1,"output":"A"}"#),
)]
#[case::responses_unknown_call_id_encodes_a_null_position(
#[case::responses_reasoning_digested(
None,
Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])),
Some(r#"{"result_of_call":null,"output":"orphan"}"#),
Some(r#"[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": "gAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAABgAAAAB"}]"#),
Some(r#"{"type":"reasoning","id":1,"encrypted_content":"sha256:010b9eb723d8d1e2"}"#),
)]
fn prompt_from_context_matches_python(
#[case] messages: Option<Value>,
#[case] input: Option<Value>,
#[case] messages: Option<&str>,
#[case] input: Option<&str>,
#[case] expected: Option<&str>,
) {
assert_eq!(
prompt_from_context(&context(messages, input)).as_deref(),
expected
);
}
/// Python `test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys`: a blank
/// text key falls through to the next one.
#[rstest]
#[case::blank_text_falls_through(
json!({"text": " ", "input_text": "fallback prompt"}),
"fallback prompt",
)]
fn prompt_from_context_skips_blank_text_keys(#[case] input: Value, #[case] expected: &str) {
assert_eq!(
prompt_from_context(&context(None, Some(input))).as_deref(),
Some(expected)
);
}
/// Where `json.dumps(..., separators=(",", ":"))` and Python `str.strip` differ from
/// `semantic.rs`: ensure_ascii escapes DEL, small floats keep Python's two-digit exponent, and
/// strip also removes the ASCII information separators.
#[rstest]
#[case::del_is_escaped(json!([{"search_results": [{"citations": "\u{7f}"}]}]), None, r#""\u007f""#)]
#[case::small_float_exponent(json!([{"search_results": [{"citations": 1.5e-7}]}]), None, "1.5e-07")]
#[case::float_at_positional_floor(json!([{"search_results": [{"citations": 1e-4}]}]), None, "0.0001")]
#[case::float_at_scientific_ceiling(json!([{"search_results": [{"citations": 1e16}]}]), None, "1e+16")]
#[case::large_float(json!([{"search_results": [{"citations": [1.25e20, -2.5, 3.0]}]}]), None, "[1.25e+20,-2.5,3.0]")]
#[case::strip_information_separators(json!([]), Some(json!("\u{1c}a\u{1f}")), "a")]
fn python_serialization_edge_cases(
#[case] messages: Value,
#[case] input: Option<Value>,
#[case] expected: &str,
) {
let actual = match input {
Some(input) => prompt_from_context(&context(None, Some(input))).unwrap_or_default(),
None => str_from_messages(messages.as_array().unwrap()),
};
assert_eq!(actual, expected);
let parse = |value: &str| serde_json::from_str::<Value>(value).unwrap();
let context = context(messages.map(parse), input.map(parse));
assert_eq!(prompt_from_context(&context).as_deref(), expected);
}
#[rstest]

100
litellm/caching/redis_semantic_cache.py
View file

@ -22,9 +22,7 @@ from litellm.constants import SEMANTIC_CACHE_EMBEDDING_TIMEOUT_SECONDS
from litellm.litellm_core_utils.asyncify import asyncify
from litellm.litellm_core_utils.prompt_templates.common_utils import (
get_str_from_messages_with_tools,
tool_call_ordinals,
tool_call_str,
tool_result_str,
get_str_from_responses_input,
)
from litellm.types.utils import EmbeddingResponse
@ -271,101 +269,7 @@ class RedisSemanticCache(BaseCache):
if "input" not in kwargs:
return None
responses_input: Final = kwargs.get("input")
prompt: Final = cls._responses_input_prompt(responses_input, cls._responses_call_ordinals(responses_input))
return prompt or None
@classmethod
def _responses_input_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> str:
prompt_parts: Final[list[str]] = []
cls._collect_responses_input_text(value, prompt_parts, call_ordinals)
return "\n".join(prompt_parts).strip()
@classmethod
def _collect_responses_input_text(
cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int]
) -> None:
value = cls._function_call_as_prompt(cls._coerce_response_input_value(value), call_ordinals)
if value is None:
return
if isinstance(value, str):
stripped_value: Final = value.strip()
if stripped_value:
prompt_parts.append(stripped_value)
return
if isinstance(value, (list, tuple)):
for item in value:
cls._collect_responses_input_text(item, prompt_parts, call_ordinals)
return
if isinstance(value, dict):
content = value.get("content")
if content is not None:
cls._collect_responses_input_text(content, prompt_parts, call_ordinals)
return
cls._collect_responses_text_fields(value, prompt_parts, call_ordinals)
return
content = getattr(value, "content", None)
if content is not None:
cls._collect_responses_input_text(content, prompt_parts, call_ordinals)
return
for text_key in ("text", "output", "input_text", "output_text"):
text_value = getattr(value, text_key, None)
if isinstance(text_value, str):
stripped_text = text_value.strip()
if stripped_text:
prompt_parts.append(stripped_text)
return
@classmethod
def _collect_responses_text_fields(
cls, value: dict, prompt_parts: list[str], call_ordinals: Mapping[str, int]
) -> None:
for text_key in ("text", "output", "input_text", "output_text"):
text_value = value.get(text_key)
if isinstance(text_value, (list, tuple)):
cls._collect_responses_input_text(text_value, prompt_parts, call_ordinals)
return
if isinstance(text_value, str) and (stripped_text := text_value.strip()):
prompt_parts.append(stripped_text)
return
@classmethod
def _responses_call_ordinals(cls, responses_input: object) -> Mapping[str, int]:
items: Final = responses_input if isinstance(responses_input, (list, tuple)) else ()
dumped_items: Final = (cls._coerce_response_input_value(item) for item in items)
return tool_call_ordinals(
item.get("call_id")
for item in dumped_items
if isinstance(item, dict) and item.get("type") == "function_call"
)
@classmethod
def _function_call_as_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> object:
if not isinstance(value, dict):
return value
if value.get("type") == "function_call":
return tool_call_str(value.get("name"), value.get("arguments"))
if value.get("type") != "function_call_output":
return value
return tool_result_str(
value.get("call_id"), call_ordinals, cls._responses_input_prompt(value.get("output"), call_ordinals)
)
@staticmethod
def _coerce_response_input_value(value: object) -> object:
model_dump: Final = getattr(value, "model_dump", None)
if callable(model_dump):
return model_dump()
dict_method: Final = getattr(value, "dict", None)
if callable(dict_method):
return dict_method()
return value
return get_str_from_responses_input(kwargs.get("input"))
def _embedding_input(self, prompt: str, router: "Router | None") -> str:
return truncate_embedding_input(

174
litellm/litellm_core_utils/prompt_templates/common_utils.py
View file

@ -2,6 +2,7 @@
Common utility functions used for translating messages across providers
"""
import hashlib
import io
import json
import mimetypes
@ -195,93 +196,148 @@ def get_str_from_messages(messages: list[AllMessageValues]) -> str:
return text
_PLAIN_TYPES: Final = frozenset({"text", "input_text", "output_text", "message"})
_PLAIN_KEYS: Final = frozenset({"role", "type", "text", "content", "status"})
_TOOL_ROLES: Final = frozenset({"tool", "function"})
_IGNORED_KEYS: Final = frozenset({"cache_control"})
_CALL_ID_KEYS: Final = frozenset({"id", "call_id", "tool_use_id", "tool_call_id"})
_OPAQUE_KEYS: Final = frozenset({"data", "file_data", "signature", "encrypted_content"})
def get_str_from_messages_with_tools(messages: object) -> str:
"""
``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns
that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``)
produce different text. Each result is encoded with the position of the call it answers, since call ids are
random per session
Semantic-cache prompt for chat-style ``messages``. Plain text parts stay plain text so text-only prompts embed
as they always have. Every other part (tool calls and results, images, unknown block types, extra fields) is kept
as compact JSON, so a new request shape changes the prompt instead of silently vanishing from it
"""
message_mappings: Final = tuple(_str_mappings(messages))
call_ordinals: Final = tool_call_ordinals(_message_tool_call_ids(message_mappings))
return "".join(_message_str_with_tools(message, call_ordinals) for message in message_mappings)
call_ordinals: Final = _call_id_ordinals(message_mappings)
return "".join(_message_prompt(message, call_ordinals) for message in message_mappings)
def tool_call_str(name: object, arguments: object) -> str:
return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}'
def get_str_from_responses_input(responses_input: object) -> str | None:
"""``get_str_from_messages_with_tools`` for Responses API ``input``, one stripped line per part"""
call_ordinals: Final = _call_id_ordinals(responses_input)
prompt: Final = "\n".join(_responses_input_parts(responses_input, call_ordinals)).strip()
return prompt or None
def tool_result_str(call_id: object, call_ordinals: Mapping[str, int], output: str) -> str:
ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None
return f'{{"result_of_call":{_compact_json(ordinal)},"output":{_compact_json(output)}}}'
def _message_prompt(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str:
role: Final = message.get("role")
if (isinstance(role, str) and role in _TOOL_ROLES) or not _is_plain(message):
return _compact_json(_normalized(message, call_ordinals))
return _plain_prompt(message, call_ordinals)
def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]:
string_ids: Final = (call_id for call_id in call_ids if isinstance(call_id, str))
return MappingProxyType({call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(string_ids), start=1)})
def _plain_prompt(value: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str:
text: Final = value.get("text")
return (text if isinstance(text, str) else "") + _content_prompt(value.get("content"), call_ordinals)
def _compact_json(value: object) -> str:
return json.dumps(value, separators=(",", ":"), default=str)
def _content_prompt(content: object, call_ordinals: Mapping[str, int]) -> str:
value: Final = _plain_value(content)
if value is None:
return ""
if isinstance(value, str):
return value
if isinstance(value, (list, tuple)):
return "".join(_content_prompt(item, call_ordinals) for item in value)
if _is_str_mapping(value) and _is_plain(value):
return _plain_prompt(value, call_ordinals)
return _compact_json(_normalized(value, call_ordinals))
def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator[object]:
for message in messages:
yield from (
block.get("id") for block in _str_mappings(message.get("content")) if block.get("type") == "tool_use"
)
yield from (tool_call.get("id") for tool_call in _str_mappings(message.get("tool_calls")))
def _responses_input_parts(value: object, call_ordinals: Mapping[str, int]) -> Iterator[str]:
item: Final = _plain_value(value)
if item is None:
return
if isinstance(item, str):
yield from _stripped(item)
elif isinstance(item, (list, tuple)):
for child in item:
yield from _responses_input_parts(child, call_ordinals)
elif _is_str_mapping(item) and _is_plain(item):
text: Final = item.get("text")
yield from _stripped(text if isinstance(text, str) else "")
yield from _responses_input_parts(item.get("content"), call_ordinals)
else:
yield _compact_json(_normalized(item, call_ordinals))
def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str:
content: Final = _content_str_with_tools(message.get("content"), call_ordinals)
def _stripped(text: str) -> Iterator[str]:
stripped: Final = text.strip()
if stripped:
yield stripped
def _is_plain(value: Mapping[str, object]) -> bool:
value_type: Final = value.get("type")
text: Final = value.get("text")
return (
(
tool_result_str(message.get("tool_call_id"), call_ordinals, content)
if message.get("role") == "tool"
else content
)
+ "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls")))
+ extract_search_results_text(message.get("search_results"))
(value_type is None or (isinstance(value_type, str) and value_type in _PLAIN_TYPES))
and (text is None or isinstance(text, str))
and all(key in _PLAIN_KEYS or key in _CALL_ID_KEYS for key in _kept_keys(value))
)
def _content_str_with_tools(content: object, call_ordinals: Mapping[str, int]) -> str:
if isinstance(content, str):
return content
return "".join(_block_str_with_tools(block, call_ordinals) for block in _str_mappings(content))
def _kept_keys(value: Mapping[str, object]) -> Iterator[str]:
return (key for key, item in value.items() if key not in _IGNORED_KEYS and not _is_empty(item))
def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str:
block_type: Final = block.get("type")
if block_type == "tool_use":
return tool_call_str(block.get("name"), block.get("input"))
if block_type == "tool_result":
return tool_result_str(
block.get("tool_use_id"), call_ordinals, _content_str_with_tools(block.get("content"), call_ordinals)
)
text: Final = block.get("text")
return text if isinstance(text, str) else ""
def _normalized(value: object, call_ordinals: Mapping[str, int], key: str | None = None) -> object:
item: Final = _plain_value(value)
if _is_str_mapping(item):
return MappingProxyType({kept: _normalized(item[kept], call_ordinals, kept) for kept in _kept_keys(item)})
if isinstance(item, (list, tuple)):
return tuple(_normalized(child, call_ordinals) for child in item)
if not isinstance(item, str):
return item
if key in _CALL_ID_KEYS:
return call_ordinals.get(item, item)
if key in _OPAQUE_KEYS or item.startswith("data:"):
return f"sha256:{hashlib.sha256(item.encode('utf-8', 'surrogatepass')).hexdigest()[:16]}"
return item
def _openai_tool_call_str(tool_call: Mapping[str, object]) -> str:
function: Final = _as_str_mapping(tool_call.get("function"))
if function is None:
return tool_call_str(None, None)
return tool_call_str(function.get("name"), function.get("arguments"))
def _call_id_ordinals(value: object) -> Mapping[str, int]:
return MappingProxyType(
{call_id: ordinal for ordinal, call_id in enumerate(dict.fromkeys(_call_ids(value)), start=1)}
)
def _call_ids(value: object, key: str | None = None) -> Iterator[str]:
item: Final = _plain_value(value)
if isinstance(item, str):
if key in _CALL_ID_KEYS:
yield item
elif _is_str_mapping(item):
for child_key, child in item.items():
if child_key not in _IGNORED_KEYS:
yield from _call_ids(child, child_key)
elif isinstance(item, (list, tuple)):
for child in item:
yield from _call_ids(child)
def _is_empty(value: object) -> bool:
return value is None or (isinstance(value, (str, list, tuple, Mapping)) and not value)
def _compact_json(value: object) -> str:
return json.dumps(value, separators=(",", ":"), default=_json_default)
def _json_default(value: object) -> object:
return dict(value) if isinstance(value, MappingProxyType) else str(value)
def _plain_value(value: object) -> object:
return value.model_dump() if isinstance(value, BaseModel) else value
def _str_mappings(values: object) -> Iterator[Mapping[str, object]]:
items: Final = values if isinstance(values, (list, tuple)) else ()
return (mapping for item in items if (mapping := _as_str_mapping(item)) is not None)
def _as_str_mapping(value: object) -> Mapping[str, object] | None:
if isinstance(value, BaseModel):
return value.model_dump()
if _is_str_mapping(value):
return value
return None
return (mapping for item in items if _is_str_mapping(mapping := _plain_value(item)))
def _is_str_mapping(value: object) -> TypeIs[Mapping[str, object]]: # guard-ok: message and block keys are str

128
tests/unit/caching/test_redis_semantic_cache.py
View file

@ -531,7 +531,7 @@ def test_redis_semantic_cache_get_cache_uses_responses_string_input():
)
def test_redis_semantic_cache_set_cache_flattens_structured_responses_input():
def test_redis_semantic_cache_set_cache_embeds_every_part_of_structured_responses_input():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
redis_semantic_cache = RedisSemanticCache.__new__(RedisSemanticCache)
@ -561,7 +561,7 @@ def test_redis_semantic_cache_set_cache_flattens_structured_responses_input():
)
redis_semantic_cache.llmcache.store.assert_called_once_with(
"What is the capital of France?\nAnswer briefly.",
'What is the capital of France?\nAnswer briefly.\n{"type":"input_image","image_url":"https://example.com/paris.png"}',
"{'content': 'Paris'}",
vector=[0.1, 0.2, 0.3],
filters={RedisSemanticCache.CACHE_KEY_FIELD_NAME: "test_key"},
@ -593,10 +593,11 @@ def test_redis_semantic_cache_prompt_extraction_keeps_tool_turns_distinct():
]
assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == (
'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1,"output":"ok"}'
'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}'
'{"type":"tool_result","tool_use_id":1,"content":"ok"}'
)
assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == (
'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1,"output":"ok"}'
assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) != (
RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd"))
)
@ -612,7 +613,8 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls()
)
assert prompt == (
'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1,"output":"ok"}'
'update the config\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}'
'\n{"type":"function_call_output","call_id":1,"output":"ok"}'
)
@ -632,30 +634,13 @@ def test_redis_semantic_cache_prompt_extraction_keeps_structured_function_call_o
]
)
expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}'
assert (
prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1,"output":"wrote 5 bytes"}}'
assert prompt_for("wrote 5 bytes") == (
'write hello\n{"type":"function_call","call_id":1,"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}\n'
'{"type":"function_call_output","call_id":1,"output":[{"type":"input_text","text":"wrote 5 bytes"}]}'
)
assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError")
def test_redis_semantic_cache_prompt_extraction_joins_multi_part_function_call_output_lines():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
prompt = RedisSemanticCache._get_prompt_from_kwargs(
input=[
{"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"},
{
"type": "function_call_output",
"call_id": "c1",
"output": [{"type": "input_text", "text": " line one "}, {"type": "input_text", "text": "line two"}],
},
]
)
assert prompt == '{"name":"run","arguments":"{}"}\n{"result_of_call":1,"output":"line one\\nline two"}'
def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
@ -670,80 +655,63 @@ def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_ans
)
assert prompt_for("c2", "c1") == (
'{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n'
'{"result_of_call":2,"output":"empty"}\n{"result_of_call":1,"output":"secret"}'
'{"type":"function_call","call_id":1,"name":"read","arguments":"a"}\n'
'{"type":"function_call","call_id":2,"name":"read","arguments":"b"}\n'
'{"type":"function_call_output","call_id":2,"output":"empty"}\n'
'{"type":"function_call_output","call_id":1,"output":"secret"}'
)
assert prompt_for("c1", "c2") != prompt_for("c2", "c1")
def test_redis_semantic_cache_prompt_extraction_handles_model_objects():
def test_redis_semantic_cache_prompt_extraction_replaces_encrypted_reasoning_with_a_digest():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
class ModelDumpInput:
def model_dump(self):
return {"content": [{"text": "model dump prompt"}]}
def prompt_for(encrypted_content: str) -> str | None:
return RedisSemanticCache._get_prompt_from_kwargs(
input=[{"type": "reasoning", "id": "rs_1", "summary": [], "encrypted_content": encrypted_content}]
)
class DictInput:
def dict(self):
return {"content": [{"output_text": "dict prompt"}]}
blob = "gAAAAB" * 100
prompt = prompt_for(blob)
assert prompt is not None
assert blob not in prompt
assert prompt.startswith('{"type":"reasoning","id":1,"encrypted_content":"sha256:')
assert prompt != prompt_for(blob + "x")
def test_redis_semantic_cache_prompt_extraction_handles_model_objects():
from pydantic import BaseModel
from litellm.caching.redis_semantic_cache import RedisSemanticCache
class ResponseInputMessage(BaseModel):
content: list[dict[str, str]]
prompt = RedisSemanticCache._get_prompt_from_kwargs(
input=[
ModelDumpInput(),
DictInput(),
{"content": [{"input_text": "inline prompt"}]},
{"content": [{"type": "input_image", "image_url": "https://example.com"}]},
ResponseInputMessage(content=[{"type": "input_text", "text": "model dump prompt"}]),
{"content": [{"type": "output_text", "text": "inline prompt", "annotations": []}]},
]
)
assert prompt == "model dump prompt\ndict prompt\ninline prompt"
assert prompt == "model dump prompt\ninline prompt"
def test_redis_semantic_cache_prompt_extraction_returns_none_without_text():
def test_redis_semantic_cache_prompt_extraction_keeps_image_only_input():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
assert RedisSemanticCache._get_prompt_from_kwargs(
input=[{"type": "input_image", "image_url": "https://example.com/a.png"}]
) == '{"type":"input_image","image_url":"https://example.com/a.png"}'
def test_redis_semantic_cache_prompt_extraction_returns_none_without_input():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
assert RedisSemanticCache._get_prompt_from_kwargs() is None
assert RedisSemanticCache._get_prompt_from_kwargs(input=None) is None
assert RedisSemanticCache._get_prompt_from_kwargs(input=" ") is None
assert (
RedisSemanticCache._get_prompt_from_kwargs(
input=[{"type": "input_image", "image_url": "https://example.com"}]
)
is None
)
def test_redis_semantic_cache_prompt_extraction_skips_blank_dict_text_keys():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
prompt = RedisSemanticCache._get_prompt_from_kwargs(
input={"text": " ", "input_text": "fallback prompt"}
)
assert prompt == "fallback prompt"
def test_redis_semantic_cache_prompt_extraction_skips_blank_object_text_keys():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
class ResponseInput:
text = " "
input_text = "fallback prompt"
prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput())
assert prompt == "fallback prompt"
def test_redis_semantic_cache_prompt_extraction_handles_object_content():
from litellm.caching.redis_semantic_cache import RedisSemanticCache
class ResponseInput:
content = [{"text": "object content prompt"}]
prompt = RedisSemanticCache._get_prompt_from_kwargs(input=ResponseInput())
assert prompt == "object content prompt"
assert RedisSemanticCache._get_prompt_from_kwargs(input=[{"type": "input_text", "text": " "}]) is None
def test_redis_semantic_cache_set_cache_skips_blank_responses_input():

126
tests/unit/litellm_core_utils/prompt_templates/test_litellm_core_utils_prompt_templates_common_utils.py
View file

@ -3,6 +3,7 @@ import functools
import json
import os
import sys
from collections.abc import Callable
from typing import Final
import pytest
@ -2023,12 +2024,12 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
"content": [{"type": "tool_use", "id": "t1", "name": "Bash", "input": {"cmd": "ls"}}],
},
],
'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}',
id="anthropic-tool-use-name-and-input-without-id",
'fix the failing test{"type":"tool_use","id":1,"name":"Bash","input":{"cmd":"ls"}}',
id="anthropic-tool-use-with-its-call-position",
),
pytest.param(
[_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}],
'fix the failing test{"result_of_call":null,"output":"calc.py"}',
'fix the failing test{"type":"tool_result","tool_use_id":1,"content":"calc.py"}',
id="anthropic-string-tool-result",
),
pytest.param(
@ -2045,7 +2046,7 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
],
}
],
'{"result_of_call":null,"output":"ab"}next',
'{"type":"tool_result","tool_use_id":1,"content":[{"type":"text","text":"a"},{"type":"text","text":"b"}]}next',
id="anthropic-nested-text-tool-result-then-text",
),
pytest.param(
@ -2061,9 +2062,11 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
},
{"role": "tool", "tool_call_id": "c1", "content": "ok"},
],
'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}'
'{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1,"output":"ok"}',
id="openai-tool-calls-in-order-before-tool-result",
'fix the failing test{"role":"assistant","content":"writing","tool_calls":['
'{"id":1,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}},'
'{"id":2,"type":"function","function":{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}}]}'
'{"role":"tool","tool_call_id":1,"content":"ok"}',
id="openai-tool-calls-then-tool-result",
),
pytest.param(
[
@ -2082,8 +2085,9 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
],
},
],
'{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}'
'{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}',
'{"type":"tool_use","id":1,"name":"Read","input":{"path":"a"}}'
'{"type":"tool_use","id":2,"name":"Read","input":{"path":"b"}}'
'{"type":"tool_result","tool_use_id":2,"content":"B"}{"type":"tool_result","tool_use_id":1,"content":"A"}',
id="anthropic-parallel-tool-results-tagged-with-their-call",
),
pytest.param(
@ -2093,19 +2097,11 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
"content": None,
"tool_calls": [{"id": "call_0", "type": "function", "function": {"name": "a", "arguments": "{}"}}],
},
{
"role": "assistant",
"content": None,
"tool_calls": [
{"id": "call_0", "type": "function", "function": {"name": "b", "arguments": "{}"}},
{"id": "call_1", "type": "function", "function": {"name": "c", "arguments": "{}"}},
],
},
{"role": "tool", "tool_call_id": "call_1", "content": "C"},
{"role": "tool", "tool_call_id": "call_0", "content": "C"},
],
'{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}'
'{"result_of_call":2,"output":"C"}',
id="reused-call-ids-keep-first-position",
'{"role":"assistant","tool_calls":[{"id":1,"type":"function","function":{"name":"a","arguments":"{}"}}]}'
'{"role":"tool","tool_call_id":1,"content":"C"}',
id="null-content-dropped-from-encoded-message",
),
pytest.param(
[
@ -2116,13 +2112,13 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
"search_results": [{"source": "s", "title": "t"}],
}
],
'{"result_of_call":null,"output":"small"}st',
id="tool-result-search-results-follow-the-encoded-result",
'{"role":"tool","tool_call_id":1,"content":"small","search_results":[{"source":"s","title":"t"}]}',
id="tool-result-keeps-search-results",
),
pytest.param(
[{"role": "tool", "tool_call_id": "c1", "content": '"},{"result_of_call":2,"output":"'}],
'{"result_of_call":null,"output":"\\"},{\\"result_of_call\\":2,\\"output\\":\\""}',
id="tool-output-cannot-forge-an-encoded-result",
[{"role": "tool", "content": 'x"}{"role":"tool","content":"y'}],
'{"role":"tool","content":"x\\"}{\\"role\\":\\"tool\\",\\"content\\":\\"y"}',
id="tool-output-is-escaped-even-without-a-call-id",
),
pytest.param(
[
@ -2131,17 +2127,32 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
tool_calls=[ChatCompletionMessageToolCall(id="c1", function=Function(name="read", arguments="{}"))],
)
],
'{"name":"read","arguments":"{}"}',
'{"role":"assistant","tool_calls":[{"function":{"arguments":"{}","name":"read"},"id":1,"type":"function"}]}',
id="openai-response-message-object",
),
pytest.param(
[{"role": "assistant", "content": None, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}, "junk"],
'{"name":null,"arguments":null}',
id="malformed-tool-call-entries",
[{"role": "user", "content": [{"type": "text", "text": "hi", "cache_control": {"type": "ephemeral"}}]}],
"hi",
id="cache-control-ignored",
),
pytest.param(
[{"role": "user", "name": "alice", "content": "hi"}],
'{"role":"user","name":"alice","content":"hi"}',
id="extra-message-field-kept",
),
pytest.param(
[{"role": "user", "content": [{"type": "brand_new_block", "payload": {"k": "v"}}, {"type": "text", "text": "!"}]}],
'{"type":"brand_new_block","payload":{"k":"v"}}!',
id="unknown-block-type-kept",
),
pytest.param(
[{"role": "user", "content": [{"type": "text", "text": 3}]}],
'{"type":"text","text":3}',
id="non-string-text-kept",
),
],
)
def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[object], expected: str) -> None:
def test_get_str_from_messages_with_tools_keeps_every_part(messages: list[object], expected: str) -> None:
assert get_str_from_messages_with_tools(messages) == expected
@ -2150,25 +2161,54 @@ def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[obj
[
pytest.param([_TASK, {"role": "assistant", "content": "done"}], id="string-content"),
pytest.param(
[
{
"role": "user",
"content": [
{"type": "text", "text": "what is "},
{"type": "image_url", "image_url": {"url": "https://example.com/a.png"}},
{"type": "text", "text": "this"},
],
}
],
id="text-and-image-parts",
[{"role": "user", "content": [{"type": "text", "text": "what is "}, {"type": "text", "text": "this"}]}],
id="text-parts",
),
pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"),
],
)
def test_get_str_from_messages_with_tools_matches_get_str_from_messages_without_tools(messages: list[object]) -> None:
def test_get_str_from_messages_with_tools_matches_get_str_from_messages_for_text(messages: list[object]) -> None:
assert get_str_from_messages_with_tools(messages) == get_str_from_messages(messages) # pyright: ignore[reportArgumentType] # untyped fixtures
def _anthropic_image(data: str) -> list[object]:
return [{"role": "user", "content": [{"type": "image", "source": {"type": "base64", "data": data}}]}]
def _openai_image(url: str) -> list[object]:
return [{"role": "user", "content": [{"type": "image_url", "image_url": {"url": url}}]}]
def _thinking(signature: str) -> list[object]:
return [{"role": "assistant", "content": [{"type": "thinking", "thinking": "hmm", "signature": signature}]}]
@pytest.mark.parametrize(
("build", "blob"),
[
pytest.param(_anthropic_image, "iVBORw0KGgo" * 50, id="anthropic-base64-image"),
pytest.param(_openai_image, "data:image/png;base64," + "iVBORw0KGgo" * 50, id="openai-data-url-image"),
pytest.param(_thinking, "EqQBCkgIARABGAIiQL" * 20, id="thinking-signature"),
],
)
def test_get_str_from_messages_with_tools_replaces_opaque_blobs_with_distinct_digests(
build: Callable[[str], list[object]], blob: str
) -> None:
prompt: Final = get_str_from_messages_with_tools(build(blob))
assert blob not in prompt
assert "sha256:" in prompt
assert len(prompt) < len(blob)
assert prompt == get_str_from_messages_with_tools(build(blob))
assert prompt != get_str_from_messages_with_tools(build(blob + "A"))
def test_get_str_from_messages_with_tools_keeps_image_urls_distinct() -> None:
assert get_str_from_messages_with_tools(_openai_image("https://example.com/a.png")) != (
get_str_from_messages_with_tools(_openai_image("https://example.com/b.png"))
)
def _parallel_reads(result_for_a: str, result_for_b: str, *, call_id_prefix: str = "c") -> list[object]:
return [
{