From 215e0143110fa458cd37ed7b5065d32b23e109bf Mon Sep 17 00:00:00 2001 From: Alessandro Pignati Date: Sat, 31 Oct 2026 23:59:41 +0100 Subject: [PATCH] refactor(guardrails): match unfinished tool calls by their announced id Pruning matched calls by their position in the rebuilt list, which need not follow the order the stream gave them, and did it by deep-copying the response and writing into the model. Match on the id each call announced, build the pruned copies with model_copy, and return the response untouched when every call finished. --- .../chat/guardrail_translation/handler.py | 44 +++++++++++++------ 1 file changed, 30 insertions(+), 14 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index 3f0500dbf0a..1ac7f6ebb0a 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -14,7 +14,6 @@ Pattern Overview: This pattern can be replicated for other message formats (e.g., Anthropic). """ -import copy import json import time import uuid @@ -788,27 +787,44 @@ class OpenAIChatCompletionsHandler(BaseTranslation): return False return all(OpenAIChatCompletionsHandler._parses_as_json(arguments) for arguments in joined.values()) + @staticmethod + def _streamed_call_ids(responses_so_far: Sequence["ModelResponseStream"]) -> Mapping[tuple[int, int], str]: + """The id each (choice index, tool call index) announced itself with.""" + choices: Final = chain.from_iterable(chunk.choices for chunk in responses_so_far) + calls: Final = chain.from_iterable( + ((choice.index, call) for call in choice.delta.tool_calls or ()) for choice in choices + ) + return MappingProxyType({(index, call.index or 0): call.id for index, call in calls if call.id}) + + @staticmethod + def _choice_without(choice: "Choices", unfinished: frozenset[str]) -> "Choices": + kept: Final = [call for call in choice.message.tool_calls or () if call.id not in unfinished] + message: Final = choice.message.model_copy(update=MappingProxyType({"tool_calls": kept or None})) + return choice.model_copy(update=MappingProxyType({"message": message})) + @staticmethod def _without_unfinished_tool_calls( assembled: "ModelResponse", responses_so_far: Sequence["ModelResponseStream"] ) -> "ModelResponse": """Copy of the rebuilt response without the calls still in flight when the stream ended. - Rebuilding gives a call that never streamed any argument text an empty JSON object, which - reads to a guardrail as a real call taking no arguments rather than an unfinished one. + Rebuilding gives a call that never streamed complete arguments an empty JSON object, which + reads to a guardrail as a real call taking no arguments rather than an unfinished one. The + calls are matched by the id they announced, since the rebuilt order need not follow the + order the stream gave them. """ joined: Final = OpenAIChatCompletionsHandler._joined_streamed_arguments(responses_so_far) - pruned: Final = copy.deepcopy(assembled) - for choice in pruned.choices: - streamed = choice.message.tool_calls or () - kept = [ - call - for position, call in enumerate(streamed) - if OpenAIChatCompletionsHandler._parses_as_json(joined.get((choice.index, position), "")) - ] - if len(kept) != len(streamed): - choice.message.tool_calls = kept or None - return pruned + unfinished: Final = frozenset( + call_id + for key, call_id in OpenAIChatCompletionsHandler._streamed_call_ids(responses_so_far).items() + if not OpenAIChatCompletionsHandler._parses_as_json(joined.get(key, "")) + ) + if not unfinished: + return assembled + pruned: Final = [ + OpenAIChatCompletionsHandler._choice_without(choice, unfinished) for choice in assembled.choices + ] + return assembled.model_copy(update=MappingProxyType({"choices": pruned})) @staticmethod def _rebuild_ended_stream_per_choice(