mirror of
https://github.com/BerriAI/litellm.git
synced 2026-11-15 09:12:18 +00:00
fix(guardrails): prune unfinished tool calls per choice
The ids of the calls still in flight were flattened into one set and matched against every rebuilt choice, so an n>1 stream where two choices announced the same id lost the finished call along with the unfinished one, and its complete arguments never reached the guardrail. Match on the choice and the id together.
This commit is contained in:
parent
1a4f6c5542
commit
e3722a1a02
2 changed files with 29 additions and 5 deletions
|
|
@ -798,8 +798,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id})
|
||||
|
||||
@staticmethod
|
||||
def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices":
|
||||
kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished]
|
||||
def _choice_without(choice: "Choices", unfinished: frozenset[tuple[int, str]]) -> "Choices":
|
||||
kept: Final = [call for call in choice.message.tool_calls or () if (choice.index, call.id) not in unfinished]
|
||||
message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None}))
|
||||
return choice.model_copy(update=MappingProxyType({"message": message}))
|
||||
|
||||
|
|
@ -811,12 +811,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
|
|||
|
||||
Rebuilding gives a call that never streamed complete arguments an empty JSON object, which
|
||||
reads to a guardrail as a real call taking no arguments rather than an unfinished one. The
|
||||
calls are matched by the id they announced, since the rebuilt order need not follow the
|
||||
order the stream gave them.
|
||||
calls are matched by the choice they belong to and the id they announced, since the rebuilt
|
||||
order need not follow the order the stream gave them, and providers number tool calls per
|
||||
choice, so the same id on another choice is another call.
|
||||
"""
|
||||
joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far)
|
||||
unfinished: Final = frozenset(
|
||||
call_id
|
||||
(key[0], call_id)
|
||||
for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items()
|
||||
if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, ""))
|
||||
)
|
||||
|
|
|
|||
|
|
@ -1721,6 +1721,29 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
|
|||
)
|
||||
assert len(CountingGuardrail.calls) == 1
|
||||
|
||||
def test_an_id_in_flight_on_one_choice_does_not_prune_another(self) -> None:
|
||||
"""Providers number tool calls per choice, so the same id on another choice is another call,
|
||||
and one still in flight must not take a finished call down with it."""
|
||||
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
|
||||
|
||||
def choice(index: int, arguments: str) -> StreamingChoices:
|
||||
return StreamingChoices(
|
||||
index=index,
|
||||
delta=Delta(content="looking", tool_calls=[
|
||||
{"index": 0, "id": "call_shared", "type": "function",
|
||||
"function": {"name": "weather", "arguments": arguments}},
|
||||
]),
|
||||
finish_reason="tool_calls",
|
||||
)
|
||||
|
||||
handler: Final = OpenAIChatCompletionsHandler()
|
||||
chunks: Final = [ModelResponseStream(choices=[choice(0, '{"city": "Pa'), choice(1, '{"city": "Rome"}')])]
|
||||
assembled: Final = handler._rebuild_ended_stream_per_choice(chunks, None)
|
||||
pruned: Final = handler._without_unfinished_tool_calls(assembled, chunks)
|
||||
assert [
|
||||
[call.function.arguments for call in single.message.tool_calls or ()] for single in pruned.choices
|
||||
] == [[], ['{"city": "Rome"}']]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None:
|
||||
"""Half a JSON object is not a call a guardrail can judge, at any number of choices."""
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue