mirror of
https://github.com/BerriAI/litellm.git
synced 2026-10-09 03:18:44 +00:00
fix(caching): encode tool result position and output together so tool text cannot forge result tags
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
This commit is contained in:
kerry
parent
19bc489cee
commit
6b4b3754d2
6 changed files with 148 additions and 83 deletions
56
litellm-rust/crates/cache/src/semantic.rs
vendored
56
litellm-rust/crates/cache/src/semantic.rs
vendored
|
|
@ -84,7 +84,7 @@ impl Embedder for PreparedEmbedding {
|
|||
}
|
||||
|
||||
/// `get_str_from_messages_with_tools`: every message's content text, tool calls and tool results,
|
||||
/// then its OpenAI `tool_calls`, then its search results. Each tool result is tagged with the
|
||||
/// then its OpenAI `tool_calls`, then its search results. Each tool result is encoded with the
|
||||
/// position of the call it answers.
|
||||
pub fn str_from_messages(messages: &[Value]) -> String {
|
||||
let messages: Vec<_> = messages.iter().filter_map(Value::as_object).collect();
|
||||
|
|
@ -107,12 +107,16 @@ pub fn str_from_messages(messages: &[Value]) -> String {
|
|||
let mut text = String::new();
|
||||
for message in messages {
|
||||
if message.get("role").and_then(Value::as_str) == Some("tool") {
|
||||
text.push_str(&tool_result_tag(
|
||||
let mut output = String::new();
|
||||
push_content_text(&mut output, message.get("content"), &call_ordinals);
|
||||
text.push_str(&tool_result_json(
|
||||
message.get("tool_call_id"),
|
||||
&call_ordinals,
|
||||
&output,
|
||||
));
|
||||
} else {
|
||||
push_content_text(&mut text, message.get("content"), &call_ordinals);
|
||||
}
|
||||
push_content_text(&mut text, message.get("content"), &call_ordinals);
|
||||
if let Some(Value::Array(tool_calls)) = message.get("tool_calls") {
|
||||
for tool_call in tool_calls.iter().filter_map(Value::as_object) {
|
||||
let function = tool_call.get("function");
|
||||
|
|
@ -142,8 +146,13 @@ fn push_content_text(
|
|||
text.push_str(&tool_call_json(block.get("name"), block.get("input")));
|
||||
}
|
||||
Some("tool_result") => {
|
||||
text.push_str(&tool_result_tag(block.get("tool_use_id"), call_ordinals));
|
||||
push_content_text(text, block.get("content"), call_ordinals);
|
||||
let mut output = String::new();
|
||||
push_content_text(&mut output, block.get("content"), call_ordinals);
|
||||
text.push_str(&tool_result_json(
|
||||
block.get("tool_use_id"),
|
||||
call_ordinals,
|
||||
&output,
|
||||
));
|
||||
}
|
||||
_ => {
|
||||
if let Some(block_text) = block.get("text").and_then(Value::as_str) {
|
||||
|
|
@ -167,14 +176,21 @@ fn tool_call_ordinals<'a>(call_ids: impl Iterator<Item = &'a Value>) -> HashMap<
|
|||
ordinals
|
||||
}
|
||||
|
||||
/// `tool_result_str`: `{"result_of_call":N}` for a result answering a known call, else empty.
|
||||
fn tool_result_tag(call_id: Option<&Value>, call_ordinals: &HashMap<&str, usize>) -> String {
|
||||
call_id
|
||||
/// `tool_result_str`: `{"result_of_call":N,"output":...}`, with a `null` position when the result
|
||||
/// answers no known call.
|
||||
fn tool_result_json(
|
||||
call_id: Option<&Value>,
|
||||
call_ordinals: &HashMap<&str, usize>,
|
||||
output: &str,
|
||||
) -> String {
|
||||
let ordinal = call_id
|
||||
.and_then(Value::as_str)
|
||||
.and_then(|call_id| call_ordinals.get(call_id))
|
||||
.map_or_else(String::new, |ordinal| {
|
||||
format!("{{\"result_of_call\":{ordinal}}}")
|
||||
})
|
||||
.and_then(|call_id| call_ordinals.get(call_id));
|
||||
format!(
|
||||
"{{\"result_of_call\":{},\"output\":{}}}",
|
||||
compact_json(&json!(ordinal)),
|
||||
compact_json(&Value::String(output.to_owned())),
|
||||
)
|
||||
}
|
||||
|
||||
/// `tool_call_str`: the compact `{"name":...,"arguments":...}` a tool call contributes.
|
||||
|
|
@ -258,14 +274,16 @@ fn collect_input_text(
|
|||
return;
|
||||
}
|
||||
if map.get("type").and_then(Value::as_str) == Some("function_call_output") {
|
||||
let tag = tool_result_tag(map.get("call_id"), call_ordinals);
|
||||
if !tag.is_empty() {
|
||||
parts.push(tag);
|
||||
if let Some(output) = map.get("output") {
|
||||
collect_input_text(output, parts, call_ordinals);
|
||||
}
|
||||
return;
|
||||
let mut output_parts = Vec::new();
|
||||
if let Some(output) = map.get("output") {
|
||||
collect_input_text(output, &mut output_parts, call_ordinals);
|
||||
}
|
||||
parts.push(tool_result_json(
|
||||
map.get("call_id"),
|
||||
call_ordinals,
|
||||
python_strip(&output_parts.join("\n")),
|
||||
));
|
||||
return;
|
||||
}
|
||||
if let Some(content) = map.get("content").filter(|content| !content.is_null()) {
|
||||
collect_input_text(content, parts, call_ordinals);
|
||||
|
|
|
|||
66
litellm-rust/crates/cache/tests/semantic.rs
vendored
66
litellm-rust/crates/cache/tests/semantic.rs
vendored
|
|
@ -38,67 +38,67 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
|
|||
json!([{"role": "tool", "content": "small", "search_results": [
|
||||
{"source": "s", "title": "t", "content": [{"text": "hidden payload"}]},
|
||||
]}]),
|
||||
"smallsthidden payload",
|
||||
r#"{"result_of_call":null,"output":"small"}sthidden payload"#,
|
||||
)]
|
||||
#[case::title_only_search_result(
|
||||
json!([{"role": "tool", "content": "small", "search_results": [
|
||||
{"source": "s", "title": "long title", "content": []},
|
||||
]}]),
|
||||
"smallslong title",
|
||||
r#"{"result_of_call":null,"output":"small"}slong title"#,
|
||||
)]
|
||||
#[case::search_results_without_content(
|
||||
json!([{"role": "tool", "search_results": [{"source": "s", "title": "t"}]}]),
|
||||
"st",
|
||||
r#"{"result_of_call":null,"output":""}st"#,
|
||||
)]
|
||||
#[case::search_result_fields_in_python_order(
|
||||
json!([{"role": "tool", "content": "c", "search_results": [
|
||||
{"citations": {"enabled": true}, "content": [{"text": "body"}], "title": "t", "source": "s"},
|
||||
{"source": "s2"},
|
||||
]}]),
|
||||
r#"cstbody{"enabled":true}s2"#,
|
||||
r#"{"result_of_call":null,"output":"c"}stbody{"enabled":true}s2"#,
|
||||
)]
|
||||
#[case::null_citations_skipped(
|
||||
json!([{"role": "tool", "content": "c", "search_results": [
|
||||
{"source": "s", "citations": null},
|
||||
]}]),
|
||||
"cs",
|
||||
r#"{"result_of_call":null,"output":"c"}s"#,
|
||||
)]
|
||||
#[case::non_string_and_non_object_entries_skipped(
|
||||
json!([{"role": "tool", "content": "c", "search_results": [
|
||||
"junk",
|
||||
{"source": 1, "title": null, "content": ["junk", {"text": 3}, {"text": "kept"}]},
|
||||
]}]),
|
||||
"ckept",
|
||||
r#"{"result_of_call":null,"output":"c"}kept"#,
|
||||
)]
|
||||
#[case::non_list_search_results_skipped(
|
||||
json!([{"role": "tool", "content": "c", "search_results": {"source": "s"}}]),
|
||||
"c",
|
||||
r#"{"result_of_call":null,"output":"c"}"#,
|
||||
)]
|
||||
#[case::citations_compact_in_insertion_order(
|
||||
json!([{"role": "tool", "search_results": [
|
||||
{"citations": {"z": 1, "a": [1.5, true, null], "m": {"k": "v"}}},
|
||||
]}]),
|
||||
r#"{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#,
|
||||
r#"{"result_of_call":null,"output":""}{"z":1,"a":[1.5,true,null],"m":{"k":"v"}}"#,
|
||||
)]
|
||||
#[case::citations_ensure_ascii(
|
||||
json!([{"role": "tool", "search_results": [{"citations": ["caf\u{e9}", "\u{4e2d}"]}]}]),
|
||||
r#"["caf\u00e9","\u4e2d"]"#,
|
||||
r#"{"result_of_call":null,"output":""}["caf\u00e9","\u4e2d"]"#,
|
||||
)]
|
||||
#[case::citations_astral_chars_as_surrogate_pairs(
|
||||
json!([{"role": "tool", "search_results": [{"citations": "\u{1f600}"}]}]),
|
||||
r#""\ud83d\ude00""#,
|
||||
r#"{"result_of_call":null,"output":""}"\ud83d\ude00""#,
|
||||
)]
|
||||
#[case::citations_escapes(
|
||||
json!([{"role": "tool", "search_results": [{"citations": "q\"\\\n\t\u{1}/"}]}]),
|
||||
r#""q\"\\\n\t\u0001/""#,
|
||||
r#"{"result_of_call":null,"output":""}"q\"\\\n\t\u0001/""#,
|
||||
)]
|
||||
#[case::citations_large_float_exponent(
|
||||
json!([{"role": "tool", "search_results": [{"citations": [1e20, 1.0]}]}]),
|
||||
"[1e+20,1.0]",
|
||||
r#"{"result_of_call":null,"output":""}[1e+20,1.0]"#,
|
||||
)]
|
||||
#[case::citations_scalars(
|
||||
json!([{"role": "tool", "search_results": [{"citations": false}, {"citations": 3}]}]),
|
||||
"false3",
|
||||
r#"{"result_of_call":null,"output":""}false3"#,
|
||||
)]
|
||||
#[case::anthropic_tool_use_name_and_input_without_id(
|
||||
json!([
|
||||
|
|
@ -113,7 +113,7 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
|
|||
json!([{"role": "user", "content": [
|
||||
{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"},
|
||||
]}]),
|
||||
"calc.py",
|
||||
r#"{"result_of_call":null,"output":"calc.py"}"#,
|
||||
)]
|
||||
#[case::anthropic_nested_text_tool_result_then_text(
|
||||
json!([{"role": "user", "content": [
|
||||
|
|
@ -123,7 +123,7 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
|
|||
]},
|
||||
{"type": "text", "text": "next"},
|
||||
]}]),
|
||||
"abnext",
|
||||
r#"{"result_of_call":null,"output":"ab"}next"#,
|
||||
)]
|
||||
#[case::openai_tool_calls_in_order_before_tool_result(
|
||||
json!([
|
||||
|
|
@ -133,7 +133,7 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
|
|||
]},
|
||||
{"role": "tool", "tool_call_id": "c1", "content": "ok"},
|
||||
]),
|
||||
r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1}ok"#,
|
||||
r#"writing{"name":"write","arguments":"{\"path\": \"a\"}"}{"name":"write","arguments":"{\"path\": \"b\"}"}{"result_of_call":1,"output":"ok"}"#,
|
||||
)]
|
||||
#[case::anthropic_parallel_tool_results_tagged_with_their_call(
|
||||
json!([
|
||||
|
|
@ -146,7 +146,7 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
|
|||
{"type": "tool_result", "tool_use_id": "t1", "content": "A"},
|
||||
]},
|
||||
]),
|
||||
r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2}B{"result_of_call":1}A"#,
|
||||
r#"{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}"#,
|
||||
)]
|
||||
#[case::reused_call_ids_keep_first_position(
|
||||
json!([
|
||||
|
|
@ -159,14 +159,18 @@ fn context(messages: Option<Value>, input: Option<Value>) -> SemanticCacheContex
|
|||
]},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "C"},
|
||||
]),
|
||||
r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2}C"#,
|
||||
r#"{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}{"result_of_call":2,"output":"C"}"#,
|
||||
)]
|
||||
#[case::unknown_call_ids_left_untagged(
|
||||
#[case::unknown_call_ids_encode_a_null_position(
|
||||
json!([
|
||||
{"role": "tool", "tool_call_id": "c9", "content": "ok"},
|
||||
{"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t9", "content": "done"}]},
|
||||
]),
|
||||
"okdone",
|
||||
r#"{"result_of_call":null,"output":"ok"}{"result_of_call":null,"output":"done"}"#,
|
||||
)]
|
||||
#[case::tool_output_cannot_forge_an_encoded_result(
|
||||
json!([{"role": "tool", "tool_call_id": "c1", "content": "\"},{\"result_of_call\":2,\"output\":\""}]),
|
||||
r#"{"result_of_call":null,"output":"\"},{\"result_of_call\":2,\"output\":\""}"#,
|
||||
)]
|
||||
#[case::malformed_tool_call_entries(
|
||||
json!([{"role": "assistant", "content": null, "tool_calls": [{"id": "c1", "type": "function"}, "junk"]}]),
|
||||
|
|
@ -271,7 +275,7 @@ fn prompt_from_messages_reads_messages_only(
|
|||
{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{\"path\":\"a.yaml\"}"},
|
||||
{"type": "function_call_output", "call_id": "c1", "output": "ok"},
|
||||
])),
|
||||
Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1}\nok"),
|
||||
Some("update the config\n{\"name\":\"write_file\",\"arguments\":\"{\\\"path\\\":\\\"a.yaml\\\"}\"}\n{\"result_of_call\":1,\"output\":\"ok\"}"),
|
||||
)]
|
||||
#[case::responses_structured_function_call_output(
|
||||
None,
|
||||
|
|
@ -279,7 +283,19 @@ fn prompt_from_messages_reads_messages_only(
|
|||
{"type": "function_call", "call_id": "c1", "name": "write_file", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "c1", "output": [{"type": "input_text", "text": "denied"}]},
|
||||
])),
|
||||
Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1}\ndenied"),
|
||||
Some("{\"name\":\"write_file\",\"arguments\":\"{}\"}\n{\"result_of_call\":1,\"output\":\"denied\"}"),
|
||||
)]
|
||||
#[case::responses_multi_part_output_joined_by_lines(
|
||||
None,
|
||||
Some(json!([
|
||||
{"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"},
|
||||
{"type": "function_call_output", "call_id": "c1", "output": [
|
||||
{"type": "input_text", "text": " line one "},
|
||||
{"type": "input_text", "text": "line two"},
|
||||
]},
|
||||
])),
|
||||
Some(r#"{"name":"run","arguments":"{}"}
|
||||
{"result_of_call":1,"output":"line one\nline two"}"#),
|
||||
)]
|
||||
#[case::responses_parallel_outputs_tagged_with_their_call(
|
||||
None,
|
||||
|
|
@ -289,12 +305,12 @@ fn prompt_from_messages_reads_messages_only(
|
|||
{"type": "function_call_output", "call_id": "c2", "output": "B"},
|
||||
{"type": "function_call_output", "call_id": "c1", "output": "A"},
|
||||
])),
|
||||
Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2}\nB\n{\"result_of_call\":1}\nA"),
|
||||
Some("{\"name\":\"read\",\"arguments\":\"a\"}\n{\"name\":\"read\",\"arguments\":\"b\"}\n{\"result_of_call\":2,\"output\":\"B\"}\n{\"result_of_call\":1,\"output\":\"A\"}"),
|
||||
)]
|
||||
#[case::responses_unknown_call_id_output_untagged(
|
||||
#[case::responses_unknown_call_id_encodes_a_null_position(
|
||||
None,
|
||||
Some(json!([{"type": "function_call_output", "call_id": "c9", "output": "orphan"}])),
|
||||
Some("orphan"),
|
||||
Some(r#"{"result_of_call":null,"output":"orphan"}"#),
|
||||
)]
|
||||
fn prompt_from_context_matches_python(
|
||||
#[case] messages: Option<Value>,
|
||||
|
|
|
|||
|
|
@ -272,11 +272,15 @@ class RedisSemanticCache(BaseCache):
|
|||
return None
|
||||
|
||||
responses_input: Final = kwargs.get("input")
|
||||
prompt_parts: Final[list[str]] = []
|
||||
cls._collect_responses_input_text(responses_input, prompt_parts, cls._responses_call_ordinals(responses_input))
|
||||
prompt: Final = "\n".join(prompt_parts).strip()
|
||||
prompt: Final = cls._responses_input_prompt(responses_input, cls._responses_call_ordinals(responses_input))
|
||||
return prompt or None
|
||||
|
||||
@classmethod
|
||||
def _responses_input_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> str:
|
||||
prompt_parts: Final[list[str]] = []
|
||||
cls._collect_responses_input_text(value, prompt_parts, call_ordinals)
|
||||
return "\n".join(prompt_parts).strip()
|
||||
|
||||
@classmethod
|
||||
def _collect_responses_input_text(
|
||||
cls, value: object, prompt_parts: list[str], call_ordinals: Mapping[str, int]
|
||||
|
|
@ -341,16 +345,17 @@ class RedisSemanticCache(BaseCache):
|
|||
if isinstance(item, dict) and item.get("type") == "function_call"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _function_call_as_prompt(value: object, call_ordinals: Mapping[str, int]) -> object:
|
||||
@classmethod
|
||||
def _function_call_as_prompt(cls, value: object, call_ordinals: Mapping[str, int]) -> object:
|
||||
if not isinstance(value, dict):
|
||||
return value
|
||||
if value.get("type") == "function_call":
|
||||
return tool_call_str(value.get("name"), value.get("arguments"))
|
||||
result_tag: Final = (
|
||||
tool_result_str(value.get("call_id"), call_ordinals) if value.get("type") == "function_call_output" else ""
|
||||
if value.get("type") != "function_call_output":
|
||||
return value
|
||||
return tool_result_str(
|
||||
value.get("call_id"), call_ordinals, cls._responses_input_prompt(value.get("output"), call_ordinals)
|
||||
)
|
||||
return (result_tag, value.get("output")) if result_tag else value
|
||||
|
||||
@staticmethod
|
||||
def _coerce_response_input_value(value: object) -> object:
|
||||
|
|
|
|||
|
|
@ -199,7 +199,7 @@ def get_str_from_messages_with_tools(messages: object) -> str:
|
|||
"""
|
||||
``get_str_from_messages`` that also keeps each conversation's tool calls and tool results, so agent turns
|
||||
that differ only in their tool exchange (Anthropic ``tool_use`` / ``tool_result``, OpenAI ``tool_calls``)
|
||||
produce different text. Each result is tagged with the position of the call it answers, since call ids are
|
||||
produce different text. Each result is encoded with the position of the call it answers, since call ids are
|
||||
random per session
|
||||
"""
|
||||
message_mappings: Final = tuple(_str_mappings(messages))
|
||||
|
|
@ -211,9 +211,9 @@ def tool_call_str(name: object, arguments: object) -> str:
|
|||
return f'{{"name":{_compact_json(name)},"arguments":{_compact_json(arguments)}}}'
|
||||
|
||||
|
||||
def tool_result_str(call_id: object, call_ordinals: Mapping[str, int]) -> str:
|
||||
def tool_result_str(call_id: object, call_ordinals: Mapping[str, int], output: str) -> str:
|
||||
ordinal: Final = call_ordinals.get(call_id) if isinstance(call_id, str) else None
|
||||
return "" if ordinal is None else f'{{"result_of_call":{ordinal}}}'
|
||||
return f'{{"result_of_call":{_compact_json(ordinal)},"output":{_compact_json(output)}}}'
|
||||
|
||||
|
||||
def tool_call_ordinals(call_ids: Iterable[object]) -> Mapping[str, int]:
|
||||
|
|
@ -234,12 +234,13 @@ def _message_tool_call_ids(messages: Iterable[Mapping[str, object]]) -> Iterator
|
|||
|
||||
|
||||
def _message_str_with_tools(message: Mapping[str, object], call_ordinals: Mapping[str, int]) -> str:
|
||||
result_tag: Final = (
|
||||
tool_result_str(message.get("tool_call_id"), call_ordinals) if message.get("role") == "tool" else ""
|
||||
)
|
||||
content: Final = _content_str_with_tools(message.get("content"), call_ordinals)
|
||||
return (
|
||||
result_tag
|
||||
+ _content_str_with_tools(message.get("content"), call_ordinals)
|
||||
(
|
||||
tool_result_str(message.get("tool_call_id"), call_ordinals, content)
|
||||
if message.get("role") == "tool"
|
||||
else content
|
||||
)
|
||||
+ "".join(_openai_tool_call_str(tool_call) for tool_call in _str_mappings(message.get("tool_calls")))
|
||||
+ extract_search_results_text(message.get("search_results"))
|
||||
)
|
||||
|
|
@ -256,8 +257,8 @@ def _block_str_with_tools(block: Mapping[str, object], call_ordinals: Mapping[st
|
|||
if block_type == "tool_use":
|
||||
return tool_call_str(block.get("name"), block.get("input"))
|
||||
if block_type == "tool_result":
|
||||
return tool_result_str(block.get("tool_use_id"), call_ordinals) + _content_str_with_tools(
|
||||
block.get("content"), call_ordinals
|
||||
return tool_result_str(
|
||||
block.get("tool_use_id"), call_ordinals, _content_str_with_tools(block.get("content"), call_ordinals)
|
||||
)
|
||||
text: Final = block.get("text")
|
||||
return text if isinstance(text, str) else ""
|
||||
|
|
|
|||
|
|
@ -593,10 +593,10 @@ def test_redis_semantic_cache_prompt_extraction_keeps_tool_turns_distinct():
|
|||
]
|
||||
|
||||
assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("ls")) == (
|
||||
'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1}ok'
|
||||
'fix the failing test{"name":"Bash","arguments":{"cmd":"ls"}}{"result_of_call":1,"output":"ok"}'
|
||||
)
|
||||
assert RedisSemanticCache._get_prompt_from_kwargs(messages=turn("pwd")) == (
|
||||
'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1}ok'
|
||||
'fix the failing test{"name":"Bash","arguments":{"cmd":"pwd"}}{"result_of_call":1,"output":"ok"}'
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -612,7 +612,7 @@ def test_redis_semantic_cache_prompt_extraction_keeps_responses_function_calls()
|
|||
)
|
||||
|
||||
assert prompt == (
|
||||
'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1}\nok'
|
||||
'update the config\n{"name":"write_file","arguments":"{\\"path\\":\\"a.yaml\\"}"}\n{"result_of_call":1,"output":"ok"}'
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -633,10 +633,29 @@ def test_redis_semantic_cache_prompt_extraction_keeps_structured_function_call_o
|
|||
)
|
||||
|
||||
expected_call = '{"name":"write_file","arguments":"{\\"path\\":\\"a.txt\\"}"}'
|
||||
assert prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1}}\nwrote 5 bytes'
|
||||
assert (
|
||||
prompt_for("wrote 5 bytes") == f'write hello\n{expected_call}\n{{"result_of_call":1,"output":"wrote 5 bytes"}}'
|
||||
)
|
||||
assert prompt_for("wrote 5 bytes") != prompt_for("PermissionError")
|
||||
|
||||
|
||||
def test_redis_semantic_cache_prompt_extraction_joins_multi_part_function_call_output_lines():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
prompt = RedisSemanticCache._get_prompt_from_kwargs(
|
||||
input=[
|
||||
{"type": "function_call", "call_id": "c1", "name": "run", "arguments": "{}"},
|
||||
{
|
||||
"type": "function_call_output",
|
||||
"call_id": "c1",
|
||||
"output": [{"type": "input_text", "text": " line one "}, {"type": "input_text", "text": "line two"}],
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
assert prompt == '{"name":"run","arguments":"{}"}\n{"result_of_call":1,"output":"line one\\nline two"}'
|
||||
|
||||
|
||||
def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_answering_different_calls():
|
||||
from litellm.caching.redis_semantic_cache import RedisSemanticCache
|
||||
|
||||
|
|
@ -652,7 +671,7 @@ def test_redis_semantic_cache_prompt_extraction_tells_apart_parallel_outputs_ans
|
|||
|
||||
assert prompt_for("c2", "c1") == (
|
||||
'{"name":"read","arguments":"a"}\n{"name":"read","arguments":"b"}\n'
|
||||
'{"result_of_call":2}\nempty\n{"result_of_call":1}\nsecret'
|
||||
'{"result_of_call":2,"output":"empty"}\n{"result_of_call":1,"output":"secret"}'
|
||||
)
|
||||
assert prompt_for("c1", "c2") != prompt_for("c2", "c1")
|
||||
|
||||
|
|
|
|||
|
|
@ -2028,7 +2028,7 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
|
|||
),
|
||||
pytest.param(
|
||||
[_TASK, {"role": "user", "content": [{"type": "tool_result", "tool_use_id": "t1", "content": "calc.py"}]}],
|
||||
"fix the failing testcalc.py",
|
||||
'fix the failing test{"result_of_call":null,"output":"calc.py"}',
|
||||
id="anthropic-string-tool-result",
|
||||
),
|
||||
pytest.param(
|
||||
|
|
@ -2045,7 +2045,7 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
|
|||
],
|
||||
}
|
||||
],
|
||||
"abnext",
|
||||
'{"result_of_call":null,"output":"ab"}next',
|
||||
id="anthropic-nested-text-tool-result-then-text",
|
||||
),
|
||||
pytest.param(
|
||||
|
|
@ -2062,7 +2062,7 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
|
|||
{"role": "tool", "tool_call_id": "c1", "content": "ok"},
|
||||
],
|
||||
'fix the failing testwriting{"name":"write","arguments":"{\\"path\\": \\"a\\"}"}'
|
||||
'{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1}ok',
|
||||
'{"name":"write","arguments":"{\\"path\\": \\"b\\"}"}{"result_of_call":1,"output":"ok"}',
|
||||
id="openai-tool-calls-in-order-before-tool-result",
|
||||
),
|
||||
pytest.param(
|
||||
|
|
@ -2083,7 +2083,7 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
|
|||
},
|
||||
],
|
||||
'{"name":"Read","arguments":{"path":"a"}}{"name":"Read","arguments":{"path":"b"}}'
|
||||
'{"result_of_call":2}B{"result_of_call":1}A',
|
||||
'{"result_of_call":2,"output":"B"}{"result_of_call":1,"output":"A"}',
|
||||
id="anthropic-parallel-tool-results-tagged-with-their-call",
|
||||
),
|
||||
pytest.param(
|
||||
|
|
@ -2104,9 +2104,26 @@ _TASK: Final = {"role": "user", "content": "fix the failing test"}
|
|||
{"role": "tool", "tool_call_id": "call_1", "content": "C"},
|
||||
],
|
||||
'{"name":"a","arguments":"{}"}{"name":"b","arguments":"{}"}{"name":"c","arguments":"{}"}'
|
||||
'{"result_of_call":2}C',
|
||||
'{"result_of_call":2,"output":"C"}',
|
||||
id="reused-call-ids-keep-first-position",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
"content": "small",
|
||||
"search_results": [{"source": "s", "title": "t"}],
|
||||
}
|
||||
],
|
||||
'{"result_of_call":null,"output":"small"}st',
|
||||
id="tool-result-search-results-follow-the-encoded-result",
|
||||
),
|
||||
pytest.param(
|
||||
[{"role": "tool", "tool_call_id": "c1", "content": '"},{"result_of_call":2,"output":"'}],
|
||||
'{"result_of_call":null,"output":"\\"},{\\"result_of_call\\":2,\\"output\\":\\""}',
|
||||
id="tool-output-cannot-forge-an-encoded-result",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
Message(
|
||||
|
|
@ -2145,17 +2162,6 @@ def test_get_str_from_messages_with_tools_keeps_tool_exchange(messages: list[obj
|
|||
],
|
||||
id="text-and-image-parts",
|
||||
),
|
||||
pytest.param(
|
||||
[
|
||||
{
|
||||
"role": "tool",
|
||||
"tool_call_id": "c1",
|
||||
"content": "small",
|
||||
"search_results": [{"source": "s", "title": "t"}],
|
||||
}
|
||||
],
|
||||
id="search-results",
|
||||
),
|
||||
pytest.param([{"role": "assistant", "content": None}, {"role": "user"}], id="missing-content"),
|
||||
],
|
||||
)
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue