fix(guardrails): prune unfinished tool calls per choice

The ids of the calls still in flight were flattened into one set and matched against every rebuilt choice, so an n>1 stream where two choices announced the same id lost the finished call along with the unfinished one, and its complete arguments never reached the guardrail. Match on the choice and the id together.
This commit is contained in:
Alessandro Pignati 2026-11-03 12:46:41 +01:00
parent 1a4f6c5542
commit e3722a1a02
2 changed files with 29 additions and 5 deletions

View file

@ -798,8 +798,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id})
@staticmethod
def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices":
kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished]
def _choice_without(choice: "Choices", unfinished: frozenset[tuple[int, str]]) -> "Choices":
kept: Final = [call for call in choice.message.tool_calls or () if (choice.index, call.id) not in unfinished]
message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None}))
return choice.model_copy(update=MappingProxyType({"message": message}))
@ -811,12 +811,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation):
Rebuilding gives a call that never streamed complete arguments an empty JSON object, which
reads to a guardrail as a real call taking no arguments rather than an unfinished one. The
calls are matched by the id they announced, since the rebuilt order need not follow the
order the stream gave them.
calls are matched by the choice they belong to and the id they announced, since the rebuilt
order need not follow the order the stream gave them, and providers number tool calls per
choice, so the same id on another choice is another call.
"""
joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far)
unfinished: Final = frozenset(
call_id
(key[0], call_id)
for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items()
if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, ""))
)

View file

@ -1721,6 +1721,29 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput:
)
assert len(CountingGuardrail.calls) == 1
def test_an_id_in_flight_on_one_choice_does_not_prune_another(self) -> None:
"""Providers number tool calls per choice, so the same id on another choice is another call,
and one still in flight must not take a finished call down with it."""
from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices
def choice(index: int, arguments: str) -> StreamingChoices:
return StreamingChoices(
index=index,
delta=Delta(content="looking", tool_calls=[
{"index": 0, "id": "call_shared", "type": "function",
"function": {"name": "weather", "arguments": arguments}},
]),
finish_reason="tool_calls",
)
handler: Final = OpenAIChatCompletionsHandler()
chunks: Final = [ModelResponseStream(choices=[choice(0, '{"city": "Pa'), choice(1, '{"city": "Rome"}')])]
assembled: Final = handler._rebuild_ended_stream_per_choice(chunks, None)
pruned: Final = handler._without_unfinished_tool_calls(assembled, chunks)
assert [
[call.function.arguments for call in single.message.tool_calls or ()] for single in pruned.choices
] == [[], ['{"city": "Rome"}']]
@pytest.mark.asyncio
async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None:
"""Half a JSON object is not a call a guardrail can judge, at any number of choices."""