From e3722a1a02219093649d1924ceca683ca9dcc764 Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Tue, 3 Nov 2026 12:46:41 +0100 Subject: [PATCH] fix(guardrails): prune unfinished tool calls per choice The ids of the calls still in flight were flattened into one set and matched against every rebuilt choice, so an n>1 stream where two choices announced the same id lost the finished call along with the unfinished one, and its complete arguments never reached the guardrail. Match on the choice and the id together. --- .../chat/guardrail_translation/handler.py | 11 +++++---- .../test_openai_guardrail_handler.py | 23 +++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index d79d2ba5e3c..1d604d49faf 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -798,8 +798,8 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id}) @staticmethod - def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices": - kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished] + def _choice_without(choice: "Choices", unfinished: frozenset[tuple[int, str]]) -> "Choices": + kept: Final = [call for call in choice.message.tool_calls or () if (choice.index, call.id) not in unfinished] message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None})) return choice.model_copy(update=MappingProxyType({"message": message})) @@ -811,12 +811,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): Rebuilding gives a call that never streamed complete arguments an empty JSON object, which reads to a guardrail as a real call taking no arguments rather than an unfinished one. The - calls are matched by the id they announced, since the rebuilt order need not follow the - order the stream gave them. + calls are matched by the choice they belong to and the id they announced, since the rebuilt + order need not follow the order the stream gave them, and providers number tool calls per + choice, so the same id on another choice is another call. """ joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far) unfinished: Final = frozenset( - call_id + (key[0], call_id) for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items() if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, "")) ) diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 0e4ca539165..63267791ad9 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1721,6 +1721,29 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) assert len(CountingGuardrail.calls) == 1 + def test_an_id_in_flight_on_one_choice_does_not_prune_another(self) -> None: + """Providers number tool calls per choice, so the same id on another choice is another call, + and one still in flight must not take a finished call down with it.""" + from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + def choice(index: int, arguments: str) -> StreamingChoices: + return StreamingChoices( + index=index, + delta=Delta(content="looking", tool_calls=[ + {"index": 0, "id": "call_shared", "type": "function", + "function": {"name": "weather", "arguments": arguments}}, + ]), + finish_reason="tool_calls", + ) + + handler: Final = OpenAIChatCompletionsHandler() + chunks: Final = [ModelResponseStream(choices=[choice(0, '{"city": "Pa'), choice(1, '{"city": "Rome"}')])] + assembled: Final = handler._rebuild_ended_stream_per_choice(chunks, None) + pruned: Final = handler._without_unfinished_tool_calls(assembled, chunks) + assert [ + [call.function.arguments for call in single.message.tool_calls or ()] for single in pruned.choices + ] == [[], ['{"city": "Rome"}']] + @pytest.mark.asyncio async def test_partial_arguments_stay_out_of_what_each_choice_inspects(self) -> None: """Half a JSON object is not a call a guardrail can judge, at any number of choices."""