mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-17 09:34:48 +00:00
perf(rust-bridge): run the malformed user text rule only when a text block is not a string
The native messages projection called the Python rule on every request, so a well-formed 501-turn request paid about 2 ms for a scan that found nothing. Rust now checks the converted messages first, user text and tool_result text included, and calls the rule only when some text value is not a string.
This commit is contained in:
parent
db6801bc74
commit
74ccdd8540
2 changed files with 33 additions and 2 deletions
|
|
@ -69,6 +69,21 @@ fn project_litellm_params<'py>(
|
|||
Ok(litellm_params(fields.into_iter().chain(folded).collect()))
|
||||
}
|
||||
|
||||
fn items(value: &Value) -> impl Iterator<Item = &Value> {
|
||||
value.as_array().into_iter().flatten()
|
||||
}
|
||||
|
||||
fn non_string_text(block: &Value) -> bool {
|
||||
block.get("type").and_then(Value::as_str) == Some("text")
|
||||
&& block.get("text").is_some_and(|text| !text.is_string())
|
||||
}
|
||||
|
||||
fn has_non_string_text(messages: &Value) -> bool {
|
||||
items(messages)
|
||||
.flat_map(|message| items(&message["content"]))
|
||||
.any(|block| non_string_text(block) || items(&block["content"]).any(non_string_text))
|
||||
}
|
||||
|
||||
/// The Python side of the Messages route: projects the prepared arguments and builds the
|
||||
/// public response, chunks and exceptions.
|
||||
pub(super) struct MessagesPythonHost {
|
||||
|
|
@ -94,8 +109,12 @@ impl MessagesPythonHost {
|
|||
argument(name)?.map(|value| value.extract()).transpose()
|
||||
};
|
||||
let model = string("model")?.ok_or(ArgumentError::Required("model"))?;
|
||||
argument("messages")?.ok_or(ArgumentError::Required("messages"))?;
|
||||
let messages = self.messages(py, arguments)?;
|
||||
let sent = from_py(&argument("messages")?.ok_or(ArgumentError::Required("messages"))?)?;
|
||||
let messages = if has_non_string_text(&sent) {
|
||||
self.messages(py, arguments)?
|
||||
} else {
|
||||
sent
|
||||
};
|
||||
let fields = self.view.body_params(py, arguments, ROUTE_HOST_MODULE)?;
|
||||
let body = [
|
||||
("model".to_string(), Value::String(model.clone())),
|
||||
|
|
|
|||
|
|
@ -187,6 +187,18 @@ async def test_drop_params_drops_non_string_user_text(server: RecordingServer, a
|
|||
assert body["messages"] == [{"role": "user", "content": [{"type": "text", "text": "hi"}]}]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@BOTH_MODES
|
||||
async def test_non_string_tool_result_text_answers_400_before_io(server: RecordingServer, asynchronous: bool) -> None:
|
||||
server.expected_requests = 0
|
||||
tool_result: Final = {"type": "tool_result", "tool_use_id": "toolu_1", "content": [{"type": "text", "text": None}]}
|
||||
|
||||
with pytest.raises(litellm.BadRequestError, match=r"tool_result toolu_1 content\.0\.text: Input should be a valid string"):
|
||||
await call(server, asynchronous=asynchronous, messages=[{"role": "user", "content": [tool_result]}])
|
||||
|
||||
assert server.requests == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@BOTH_MODES
|
||||
async def test_messages_emptied_by_drop_params_answer_400_before_io(server: RecordingServer, asynchronous: bool) -> None:
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue