From f394b968c554eac04ff4933d2b71b2b9727b3f7b Mon Sep 17 00:00:00 2001 From: Jesus Velarde Date: Fri, 13 Nov 2026 16:01:27 -0800 Subject: [PATCH] fix(guardrails): write tool-call rewrites back on multi-choice chat streams The ended-stream write-back grouped tool-call fragments by tool index alone, so it refused every stream with more than one choice. The pipeline then released the original chunks, undoing text redactions on n>1 streams as soon as a guardrail also rewrote a tool call. Fragments are now keyed by (choice index, tool index), the key stream_chunk_builder already uses, and the refusal is gone --- .../chat/guardrail_translation/handler.py | 48 +++++++--------- .../test_openai_guardrail_handler.py | 55 +++++++++++++------ 2 files changed, 57 insertions(+), 46 deletions(-) diff --git a/litellm/llms/openai/chat/guardrail_translation/handler.py b/litellm/llms/openai/chat/guardrail_translation/handler.py index c5488bc8a98..3fe55f0d685 100644 --- a/litellm/llms/openai/chat/guardrail_translation/handler.py +++ b/litellm/llms/openai/chat/guardrail_translation/handler.py @@ -1225,20 +1225,14 @@ class OpenAIChatCompletionsHandler(BaseTranslation): def _function_tool_call_fragments( responses_so_far: Sequence["ModelResponseStream"], ) -> tuple[tuple[ChatCompletionDeltaToolCall, ...], ...]: - """Group the stream's function tool-call fragments by their tool-call index, in - the index order ``stream_chunk_builder`` lists the rebuilt tool calls, keeping - only the indices the builder keeps (an id and a name somewhere in the stream).""" - fragments: Final = tuple( - tool_call - for response in responses_so_far - for choice in response.choices - for tool_call in choice.delta.tool_calls or () - if isinstance(tool_call, ChatCompletionDeltaToolCall) - ) - identified: Final = frozenset(fragment.index for fragment in fragments if fragment.id) - named: Final = frozenset(fragment.index for fragment in fragments if fragment.function.name) + """Group the stream's function tool-call fragments by (choice index, tool-call index), in + the order the per-choice rebuild lists the tool calls, keeping only the calls + ``stream_chunk_builder`` keeps (an id and a name somewhere in the stream).""" + fragments: Final = tuple(_keyed_function_tool_call_fragments(responses_so_far)) + identified: Final = frozenset(key for key, fragment in fragments if fragment.id) + named: Final = frozenset(key for key, fragment in fragments if fragment.function.name) return tuple( - tuple(fragment for fragment in fragments if fragment.index == index) for index in sorted(identified & named) + tuple(fragment for key, fragment in fragments if key == kept) for kept in sorted(identified & named) ) def _write_ended_stream_tool_call_rewrites( @@ -1251,25 +1245,13 @@ class OpenAIChatCompletionsHandler(BaseTranslation): """Write ended-stream guardrail tool-call rewrites back across the buffered chunks: the rewritten name and full arguments land in the tool call's first fragment and the arguments of its later fragments are blanked, mirroring the - text write-back. A rewrite on a stream carrying more than one distinct choice - index, or whose fragments do not line up with the rebuilt tool calls, is - reported as undeliverable, so the pipeline executor discards it and releases - the original chunks.""" + text write-back. A rewrite whose fragments do not line up with the rebuilt tool + calls is reported as undeliverable, so the pipeline executor discards it and + releases the original chunks.""" post_guardrail_tool_calls: Final = self._function_tool_call_shapes(guardrailed_response) if post_guardrail_tool_calls == pre_guardrail_tool_calls: return - stream_choice_indices: Final = frozenset( - choice.index for response in responses_so_far for choice in response.choices - ) fragments_by_tool_call: Final = self._function_tool_call_fragments(responses_so_far) - if len(stream_choice_indices) != 1: - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - - raise UndeliverableStreamRewrite( - guardrail_name, - f"the stream carries {len(stream_choice_indices)} choices and tool-call rewrites are only written " - "back on single-choice streams", - ) if len(fragments_by_tool_call) != len(post_guardrail_tool_calls): from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite @@ -1555,6 +1537,16 @@ def _streamed_server_tool_calls( yield (choice.index, server_tool_call["id"]), server_tool_call +def _keyed_function_tool_call_fragments( + responses: Sequence["ModelResponseStream"], +) -> Iterator[tuple[tuple[int, int], ChatCompletionDeltaToolCall]]: + for response in responses: + for choice in response.choices: + for tool_call in choice.delta.tool_calls or (): + if isinstance(tool_call, ChatCompletionDeltaToolCall): + yield (choice.index, tool_call.index), tool_call + + def _choice_server_tool_calls( calls_by_id: Mapping[tuple[int, str | None], ChatCompletionAssistantToolCall], choice_index: int ) -> tuple[ChatCompletionAssistantToolCall, ...]: diff --git a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py index 54832bb35cc..926350050a9 100644 --- a/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py +++ b/tests/unit/llms/openai/chat/guardrail_translation/test_openai_guardrail_handler.py @@ -1559,7 +1559,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: assert chunks[4].usage.completion_tokens == 7 @staticmethod - def _two_choice_tool_call_stream_chunks() -> list: + def _two_choice_tool_call_stream_chunks(leading_texts: tuple[str, str] | None = None) -> list: from litellm.types.utils import ( ChatCompletionDeltaToolCall, Delta, @@ -1569,7 +1569,10 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ) def chunk( - choice_index: int, tool_call: ChatCompletionDeltaToolCall | None, finish_reason: Optional[str] = None + choice_index: int, + tool_call: ChatCompletionDeltaToolCall | None, + finish_reason: Optional[str] = None, + content: str | None = None, ) -> ModelResponseStream: return ModelResponseStream( id="chatcmpl-123", @@ -1579,7 +1582,7 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: choices=[ StreamingChoices( index=choice_index, - delta=Delta(tool_calls=[tool_call] if tool_call else None), + delta=Delta(content=content, tool_calls=[tool_call] if tool_call else None), finish_reason=finish_reason, ) ], @@ -1590,7 +1593,11 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: id=call_id, index=0, type="function", function=Function(name=name, arguments=arguments) ) + text_chunks: Final = ( + [chunk(index, None, content=text) for index, text in enumerate(leading_texts)] if leading_texts else [] + ) return [ + *text_chunks, chunk(0, fragment("", name="lookup_fruit", call_id="call_1")), chunk(1, fragment("", name="lookup_fruit", call_id="call_2")), chunk(0, fragment('{"fruit": "pers')), @@ -1639,25 +1646,37 @@ class TestOpenAIChatCompletionsHandlerStreamingOutput: ] == [("call_1", '{"fruit": "persimmon"}'), ("call_2", '{"fruit": "durian"}')] @pytest.mark.asyncio - async def test_deliver_ended_stream_tool_call_rewrite_on_multi_choice_stream_fails_closed(self): - from litellm.proxy.policy_engine.pipeline_executor import UndeliverableStreamRewrite - + async def test_deliver_ended_stream_rewrites_each_choices_text_and_tool_call(self): handler = OpenAIChatCompletionsHandler() - chunks = self._two_choice_tool_call_stream_chunks() + chunks = self._two_choice_tool_call_stream_chunks(leading_texts=("look up ", "check ")) - with pytest.raises(UndeliverableStreamRewrite, match="the stream carries 2 choices") as raised: - await handler.process_output_streaming_response( - responses_so_far=chunks, - guardrail_to_apply=MockGuardrail(guardrail_name="test"), - litellm_logging_obj=None, - deliver_ended_stream_rewrites=True, - ) - - assert raised.value.guardrail_name == "test" - assert raised.value.reason == ( - "the stream carries 2 choices and tool-call rewrites are only written back on single-choice streams" + await handler.process_output_streaming_response( + responses_so_far=chunks, + guardrail_to_apply=MockGuardrail(guardrail_name="test"), + litellm_logging_obj=None, + deliver_ended_stream_rewrites=True, ) + assert [ + ( + c.choices[0].index, + c.choices[0].delta.content, + [t.function.arguments for t in c.choices[0].delta.tool_calls or ()], + ) + for c in chunks + ] == [ + (0, "LOOK UP ", []), + (1, "CHECK ", []), + (0, None, ['{"fruit": "PERSIMMON"}']), + (1, None, ['{"fruit": "DURIAN"}']), + (0, None, [""]), + (1, None, [""]), + (0, None, [""]), + (1, None, [""]), + (0, None, []), + (1, None, []), + ] + @pytest.mark.asyncio async def test_deliver_ended_stream_clean_multi_choice_stream_released_untouched(self): handler = OpenAIChatCompletionsHandler()